diff --git a/NuGet.config b/NuGet.config index d869a5afb008cb..246ff89d9ad8d0 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5410594cf94918..83231298a05880 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,37 +13,37 @@ https://github.com/dotnet/wcf 7f504aabb1988e9a093c1e74d8040bd52feb2f01 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 https://github.com/dotnet/command-line-api @@ -65,18 +65,18 @@ 5850baac30805f358b113f0b12966ce234023725 - + https://github.com/dotnet/emsdk - 2b4c425892386783f059f98bd2aa8d780b78617f + a84b7393bb5e3c30caf5fc50752c732913cc1fce - + https://github.com/dotnet/emsdk - 2b4c425892386783f059f98bd2aa8d780b78617f + a84b7393bb5e3c30caf5fc50752c732913cc1fce - + https://github.com/dotnet/emsdk - 2b4c425892386783f059f98bd2aa8d780b78617f + a84b7393bb5e3c30caf5fc50752c732913cc1fce @@ -93,87 +93,87 @@ - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba https://github.com/dotnet/runtime-assets @@ -227,61 +227,61 @@ https://github.com/dotnet/runtime-assets ac1dbd1987d7ca08b7347ef51130b7d69f6cf58c - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 - + https://github.com/dotnet/llvm-project - 5a502086ee0ced380f53194c674e3c21701fb5c0 + 2966bcaf054cabeb1971447d0b5c396d57573686 https://github.com/dotnet/runtime @@ -333,9 +333,9 @@ https://github.com/dotnet/xharness 8dd9b8e6004e22c7b120af0631f0809865ff2088 - + https://github.com/dotnet/arcade - c11157d174c111c4ab316dc60ff53311e410ef97 + ca4c91e56da9ae88fc62578058d8aee531679dba https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 919a2840d4b4c4..1ff508fbcf8a73 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -83,22 +83,22 @@ 9.0.109 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 2.9.0-beta.26458.10 - 9.0.0-beta.26458.10 - 2.9.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 - 9.0.0-beta.26458.10 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 2.9.0-beta.26507.8 + 9.0.0-beta.26507.8 + 2.9.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 + 9.0.0-beta.26507.8 1.4.0 @@ -229,26 +229,26 @@ 2.5.12 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 - 9.0.21-servicing.26464.5 - 9.0.21 + 9.0.22-servicing.26508.5 + 9.0.22 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100Version) - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 - 19.1.0-alpha.1.26457.4 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 + 19.1.0-alpha.1.26505.5 3.1.7 1.0.406601 diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 8da43d3b5837a1..b0020eb392f43e 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -112,7 +112,8 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds. + # Entra-enabled Helix templates do not forward this value to their processes. - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml index 68fa739c4ab215..064580818f79b6 100644 --- a/eng/common/core-templates/steps/send-to-helix.yml +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -4,7 +4,9 @@ parameters: HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixAccessToken: '' # optional -- legacy access token; not forwarded when HelixUseEntraAuthentication is true + HelixUseEntraAuthentication: false # optional -- use refreshable Entra authentication instead of a PAT or anonymous access + HelixAzureSubscription: '' # required when HelixUseEntraAuthentication is true -- workload identity federation service connection name authorized for Helix HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY HelixProjectArguments: '' # optional -- arguments passed to the build command HelixConfiguration: '' # optional -- additional property attached to a job @@ -31,63 +33,163 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} + - ${{ if and(eq(parameters.HelixUseEntraAuthentication, true), eq(parameters.HelixAzureSubscription, '')) }}: + - pwsh: throw "HelixAzureSubscription must be set when HelixUseEntraAuthentication is true." + displayName: Validate Helix Entra authentication + condition: ${{ parameters.condition }} + + - ${{ if eq(parameters.HelixUseEntraAuthentication, true) }}: + - task: AzureCLI@2 + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + inputs: + azureSubscription: ${{ parameters.HelixAzureSubscription }} + keepAzSessionActive: true + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + # AzureCLI@2 replaces SYSTEM_ACCESSTOKEN with the service connection token. + $env:SYSTEM_ACCESSTOKEN = $env:HELIX_AZDO_ACCESSTOKEN + + $(Build.SourcesDirectory)\eng\common\msbuild.ps1 ` + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} ` + /restore ` + /p:TreatWarningsAsErrors=false ` + /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} ` + ${{ parameters.HelixProjectArguments }} ` + /t:Test ` + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + HELIX_AZDO_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - task: AzureCLI@2 + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + inputs: + azureSubscription: ${{ parameters.HelixAzureSubscription }} + keepAzSessionActive: true + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # AzureCLI@2 replaces SYSTEM_ACCESSTOKEN with the service connection token. + export SYSTEM_ACCESSTOKEN="$HELIX_AZDO_ACCESSTOKEN" + + $(Build.SourcesDirectory)/eng/common/msbuild.sh \ + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} \ + /restore \ + /p:TreatWarningsAsErrors=false \ + /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} \ + ${{ parameters.HelixProjectArguments }} \ + /t:Test \ + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + HELIX_AZDO_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + + - ${{ if eq(parameters.HelixUseEntraAuthentication, false) }}: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b25e9fa..00000000000000 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/global.json b/global.json index fc0abba4c41317..fdf978247a13a7 100644 --- a/global.json +++ b/global.json @@ -1,16 +1,16 @@ { "sdk": { - "version": "9.0.116", + "version": "9.0.121", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "9.0.116" + "dotnet": "9.0.121" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26458.10", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26458.10", - "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.26458.10", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.26507.8", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.26507.8", + "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.26507.8", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "9.0.0-rtm.24511.16" diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Mutex.CoreCLR.Unix.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Mutex.CoreCLR.Unix.cs index 52233ffc583b14..ddbb84fe547eb7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Mutex.CoreCLR.Unix.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Mutex.CoreCLR.Unix.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -14,9 +15,15 @@ namespace System.Threading /// public sealed partial class Mutex : WaitHandle { + [FeatureSwitchDefinition("System.Threading.Mutex.UnsafeDefaultMutexScopeGlobal")] + private static bool UnsafeDefaultMutexScopeGlobal { get; } = + AppContext.TryGetSwitch("System.Threading.Mutex.UnsafeDefaultMutexScopeGlobal", out bool enabled) && enabled; + private void CreateMutexCore(bool initiallyOwned, string? name, out bool createdNew) { - SafeWaitHandle mutexHandle = CreateMutexCore(initiallyOwned, name, out int errorCode, out string? errorDetails); + bool isUserScope = !string.IsNullOrEmpty(name) && !UnsafeDefaultMutexScopeGlobal; + SafeWaitHandle mutexHandle = + CreateMutexCore(initiallyOwned, name, isUserScope, out int errorCode, out string? errorDetails); if (mutexHandle.IsInvalid) { mutexHandle.SetHandleAsInvalid(); @@ -42,7 +49,8 @@ private static OpenExistingResult OpenExistingWorker(string name, out Mutex? res // with parameters to allow us to view & edit the ACL. This will // fail if we don't have permission to view or edit the ACL's. // If that happens, ask for less permissions. - SafeWaitHandle myHandle = OpenMutexCore(name, out int errorCode, out string? errorDetails); + SafeWaitHandle myHandle = + OpenMutexCore(name, isUserScope: !UnsafeDefaultMutexScopeGlobal, out int errorCode, out string? errorDetails); if (myHandle.IsInvalid) { @@ -86,11 +94,13 @@ public void ReleaseMutex() private static unsafe SafeWaitHandle CreateMutexCore( bool initialOwner, string? name, + bool isUserScope, out int errorCode, out string? errorDetails) { byte* systemCallErrors = stackalloc byte[SystemCallErrorsBufferSize]; - SafeWaitHandle mutexHandle = CreateMutex(initialOwner, name, systemCallErrors, SystemCallErrorsBufferSize); + SafeWaitHandle mutexHandle = + CreateMutex(initialOwner, name, isUserScope, systemCallErrors, SystemCallErrorsBufferSize); // Get the error code even if the handle is valid, as it could be ERROR_ALREADY_EXISTS, indicating that the mutex // already exists and was opened @@ -100,10 +110,14 @@ private static unsafe SafeWaitHandle CreateMutexCore( return mutexHandle; } - private static unsafe SafeWaitHandle OpenMutexCore(string name, out int errorCode, out string? errorDetails) + private static unsafe SafeWaitHandle OpenMutexCore( + string name, + bool isUserScope, + out int errorCode, + out string? errorDetails) { byte* systemCallErrors = stackalloc byte[SystemCallErrorsBufferSize]; - SafeWaitHandle mutexHandle = OpenMutex(name, systemCallErrors, SystemCallErrorsBufferSize); + SafeWaitHandle mutexHandle = OpenMutex(name, isUserScope, systemCallErrors, SystemCallErrorsBufferSize); errorCode = mutexHandle.IsInvalid ? Marshal.GetLastPInvokeError() : Interop.Errors.ERROR_SUCCESS; errorDetails = mutexHandle.IsInvalid ? GetErrorDetails(systemCallErrors) : null; return mutexHandle; @@ -127,9 +141,18 @@ private static unsafe SafeWaitHandle OpenMutexCore(string name, out int errorCod } [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "PAL_CreateMutexW", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] - private static unsafe partial SafeWaitHandle CreateMutex([MarshalAs(UnmanagedType.Bool)] bool initialOwner, string? name, byte* systemCallErrors, uint systemCallErrorsBufferSize); + private static unsafe partial SafeWaitHandle CreateMutex( + [MarshalAs(UnmanagedType.Bool)] bool initialOwner, + string? name, + [MarshalAs(UnmanagedType.Bool)] bool isUserScope, + byte* systemCallErrors, + uint systemCallErrorsBufferSize); [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "PAL_OpenMutexW", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] - private static unsafe partial SafeWaitHandle OpenMutex(string name, byte* systemCallErrors, uint systemCallErrorsBufferSize); + private static unsafe partial SafeWaitHandle OpenMutex( + string name, + [MarshalAs(UnmanagedType.Bool)] bool isUserScope, + byte* systemCallErrors, + uint systemCallErrorsBufferSize); } } diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index 441f45f96cff34..08d1a384fb842a 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -932,6 +932,7 @@ PALAPI PAL_CreateMutexW( IN BOOL bInitialOwner, IN LPCWSTR lpName, + IN BOOL bCurrentUserOnly, IN LPSTR lpSystemCallErrors, IN DWORD dwSystemCallErrorsBufferSize); @@ -953,6 +954,7 @@ HANDLE PALAPI PAL_OpenMutexW( IN LPCWSTR lpName, + IN BOOL bCurrentUserOnly, IN LPSTR lpSystemCallErrors, IN DWORD dwSystemCallErrorsBufferSize); diff --git a/src/coreclr/pal/inc/pal_error.h b/src/coreclr/pal/inc/pal_error.h index b387e68540060a..b03f9cefa129f5 100644 --- a/src/coreclr/pal/inc/pal_error.h +++ b/src/coreclr/pal/inc/pal_error.h @@ -147,7 +147,6 @@ #define ERROR_PALINIT_TLS 65295L #define ERROR_PALINIT_ENV 65296L #define ERROR_PALINIT_DBG_CHANNELS 65297L -#define ERROR_PALINIT_SHARED_MEMORY_MANAGER 65298L #define ERROR_PALINIT_SHM 65299L #define ERROR_PALINIT_MODULE_MANAGER 65300L diff --git a/src/coreclr/pal/src/include/pal/mutex.hpp b/src/coreclr/pal/src/include/pal/mutex.hpp index 016668dafb1624..4bff5fe64caa2a 100644 --- a/src/coreclr/pal/src/include/pal/mutex.hpp +++ b/src/coreclr/pal/src/include/pal/mutex.hpp @@ -37,6 +37,7 @@ namespace CorUnix LPSECURITY_ATTRIBUTES lpMutexAttributes, BOOL bInitialOwner, LPCSTR lpName, + BOOL bCurrentUserOnly, HANDLE *phMutex ); @@ -51,6 +52,7 @@ namespace CorUnix SharedMemorySystemCallErrors *errors, CPalThread *pThread, LPCSTR lpName, + BOOL bCurrentUserOnly, HANDLE *phMutex ); @@ -76,8 +78,13 @@ Design find any other sync object with a timed wait with the necessary properties, so polling is done for timed waits. Shared memory files -- Session-scoped mutexes (name not prefixed, or prefixed with Local) go in /tmp/.dotnet/shm/session/ -- Globally-scoped mutexes (name prefixed with Global) go in /tmp/.dotnet/shm/global/ +- Session-scoped mutexes (name not prefixed, or prefixed with Local) go in + /{.dotnet|.dotnet-uid}/shm/session/ +- Globally-scoped mutexes (name prefixed with Global) go in + /{.dotnet|.dotnet-uid}/shm/global/ +- User-unscoped mutexes use the global shared-files base. User-scoped mutexes use a user-specific system base. +- In an Apple application-group container, user-scoped mutexes use .dotnet-uid in parallel to the .dotnet directory used by + user-unscoped mutexes, preserving the distinction between the scopes within the already-user-specific container. - Contains shared state, and is mmap'ped into the process, see SharedMemorySharedDataHeader and NamedMutexSharedData for data stored - Creation and deletion is synchronized using an exclusive file lock on the shm directory @@ -94,7 +101,7 @@ Shared memory files Lock files when using file locks: - In addition to the shared memory file, we need another file for the actual synchronization file lock, since a file lock on the shared memory file is used for lifetime purposes. -- These files go in /tmp/.dotnet/lockfiles/session|global/ +- These files go in /{.dotnet|.dotnet-uid}/lockfiles/session|global/ - The file is empty, and is only used for file locks Process data @@ -216,10 +223,10 @@ class NamedMutexProcessData : public SharedMemoryProcessDataBase bool m_hasRefFromLockOwnerThread; public: - static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, bool acquireLockIfCreated, bool *createdRef); - static SharedMemoryProcessDataHeader *Open(SharedMemorySystemCallErrors *errors, LPCSTR name); + static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, bool isUserScope, bool acquireLockIfCreated, bool *createdRef); + static SharedMemoryProcessDataHeader *Open(SharedMemorySystemCallErrors *errors, LPCSTR name, bool isUserScope); private: - static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, bool createIfNotExist, bool acquireLockIfCreated, bool *createdRef); + static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, bool isUserScope, bool createIfNotExist, bool acquireLockIfCreated, bool *createdRef); public: NamedMutexProcessData( diff --git a/src/coreclr/pal/src/include/pal/sharedmemory.h b/src/coreclr/pal/src/include/pal/sharedmemory.h index fce6d2859cc05e..8fd104ce69a6ad 100644 --- a/src/coreclr/pal/src/include/pal/sharedmemory.h +++ b/src/coreclr/pal/src/include/pal/sharedmemory.h @@ -11,38 +11,45 @@ #define static_assert_no_msg( cond ) static_assert( cond, #cond ) #endif // !static_assert_no_msg -// The folder used for storing shared memory files and their lock files is defined in -// the gSharedFilesPath global variable. The value of the variable depends on which -// OS is being used, and if the application is running in a sandbox in Mac. -// gSharedFilesPath ends with '/' -// - Global shared memory files go in: -// {gSharedFilesPath}/.dotnet/shm/global/ -// - Session-scoped shared memory files go in: -// {gSharedFilesPath}/.dotnet/shm/session/ -// - Lock files associated with global shared memory files go in: -// {gSharedFilesPath}/.dotnet/lockfiles/global/ -// - Lock files associated with session-scoped shared memory files go in: -// {gSharedFilesPath}/.dotnet/lockfiles/session/ +// User-unscoped files use gSharedFilesPath as their system base. On Apple, user-scoped files use the configured +// application-group container when present, otherwise a non-empty TMPDIR. On Linux, they use a non-empty XDG_RUNTIME_DIR. +// All other cases use ~/.dotnet/corefx/Runtime/SharedMemory. +// In an Apple application-group container, user-scoped files use .dotnet-uid to keep them separate from user-unscoped +// files in .dotnet. In all other cases, both scopes use .dotnet under their respective system bases. +// - Shared memory files go in: +// /{.dotnet|.dotnet-uid}/shm/{global|session}/ +// - Lock files go in: +// /{.dotnet|.dotnet-uid}/lockfiles/{global|session}/ #define SHARED_MEMORY_MAX_FILE_NAME_CHAR_COUNT (_MAX_FNAME - 1) #define SHARED_MEMORY_MAX_NAME_CHAR_COUNT (STRING_LENGTH("Global\\") + SHARED_MEMORY_MAX_FILE_NAME_CHAR_COUNT) #define SHARED_MEMORY_RUNTIME_TEMP_DIRECTORY_NAME ".dotnet" -#define SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME ".dotnet/shm" -#define SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME ".dotnet/lockfiles" -static_assert_no_msg(ARRAY_SIZE(SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME) >= ARRAY_SIZE(SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME)); +#define SHARED_MEMORY_USER_RUNTIME_TEMP_DIRECTORY_NAME ".dotnet-uid" +#define SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME "shm" +#define SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME "lockfiles" +static_assert_no_msg(STRING_LENGTH(SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME) >= STRING_LENGTH(SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME)); #define SHARED_MEMORY_GLOBAL_DIRECTORY_NAME "global" #define SHARED_MEMORY_SESSION_DIRECTORY_NAME_PREFIX "session" -static_assert_no_msg(ARRAY_SIZE(SHARED_MEMORY_SESSION_DIRECTORY_NAME_PREFIX) >= ARRAY_SIZE(SHARED_MEMORY_GLOBAL_DIRECTORY_NAME)); -#define SHARED_MEMORY_UNIQUE_TEMP_NAME_TEMPLATE ".coreclr.XXXXXX" +#define SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_0 ".dotnet" +#define SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_1 "corefx" +#define SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_2 "Runtime" +#define SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_3 "SharedMemory" +#define SHARED_MEMORY_PERSISTED_USER_DIRECTORY_SUFFIX \ + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_0 "/" \ + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_1 "/" \ + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_2 "/" \ + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_3 #define SHARED_MEMORY_MAX_SESSION_ID_CHAR_COUNT (10) // Note that this Max size does not include the prefix folder path size which is unknown (in the case of sandbox) until runtime #define SHARED_MEMORY_MAX_FILE_PATH_CHAR_COUNT \ ( \ + STRING_LENGTH(SHARED_MEMORY_USER_RUNTIME_TEMP_DIRECTORY_NAME) + \ + 1 /* path separator */ + \ STRING_LENGTH(SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME) + \ 1 /* path separator */ + \ STRING_LENGTH(SHARED_MEMORY_SESSION_DIRECTORY_NAME_PREFIX) + \ @@ -98,12 +105,20 @@ class SharedMemorySystemCallErrors void Append(LPCSTR format, ...); }; +class SharedMemoryId; + class SharedMemoryHelpers { + friend class SharedMemoryManager; + private: - static const mode_t PermissionsMask_CurrentUser_ReadWriteExecute; + static const mode_t PermissionsMask_OwnerUser_ReadWrite; + static const mode_t PermissionsMask_OwnerUser_ReadWriteExecute; + static const mode_t PermissionsMask_NonOwnerUsers_Write; static const mode_t PermissionsMask_AllUsers_ReadWrite; static const mode_t PermissionsMask_AllUsers_ReadWriteExecute; + static const mode_t PermissionsMask_Sticky; + static const int TransientRetryCount; public: static const UINT32 InvalidProcessId; static const SIZE_T InvalidThreadId; @@ -115,20 +130,53 @@ class SharedMemoryHelpers static void *Alloc(SIZE_T byteCount); - template static void BuildSharedFilesPath(PathCharString& destination, const char (&suffix)[SuffixByteCount]); - static void BuildSharedFilesPath(PathCharString& destination, const char *suffix, int suffixByteCount); + static void BuildSharedFilesSystemPath(PathCharString& destination, const SharedMemoryId *id); + static void BuildRuntimeRootPath(PathCharString& destination, const SharedMemoryId *id); + static void BuildSharedMemoryDirectoryPath(PathCharString& destination, const SharedMemoryId *id); + static void BuildLockFilesDirectoryPath(PathCharString& destination, const SharedMemoryId *id); + static void BuildSessionDirectoryPath(PathCharString& destination, const SharedMemoryId *id, bool isLockFile); + static void BuildPath(PathCharString& destination, const SharedMemoryId *id, bool isLockFile); static bool AppendUInt32String(PathCharString& destination, UINT32 value); - static bool EnsureDirectoryExists(SharedMemorySystemCallErrors *errors, const char *path, bool isGlobalLockAcquired, bool createIfNotExist = true, bool isSystemDirectory = false); + static int OpenSharedFilesSystemDirectory(SharedMemorySystemCallErrors *errors, const SharedMemoryId *id); + static int EnsureDirectoryExists( + SharedMemorySystemCallErrors *errors, + int parentDirectoryFileDescriptor, + LPCSTR directoryName, + LPCSTR directoryPathForError, + const SharedMemoryId *id, + bool createIfNotExist = true); private: + static bool AppendPathComponent(PathCharString& path, LPCSTR component); + static void BuildUserSpecificSharedFilesPath( + PathCharString& destination, + uid_t uid, + SIZE_T *persistedUserHomeDirectoryPathCharCountRef, + bool *isApplicationContainerRef); + static void EnsurePersistedUserDirectoryExists( + SharedMemorySystemCallErrors *errors, + LPCSTR path, + SIZE_T homeDirectoryPathCharCount, + const SharedMemoryId *id); static int Open(SharedMemorySystemCallErrors *errors, LPCSTR path, int flags, mode_t mode = static_cast(0)); + static int OpenAt( + SharedMemorySystemCallErrors *errors, + int directoryFileDescriptor, + LPCSTR path, + int flags, + mode_t mode = static_cast(0)); public: static int OpenDirectory(SharedMemorySystemCallErrors *errors, LPCSTR path); - static int CreateOrOpenFile(SharedMemorySystemCallErrors *errors, LPCSTR path, bool createIfNotExist = true, bool *createdRef = nullptr); + static int CreateOrOpenFile( + SharedMemorySystemCallErrors *errors, + int directoryFileDescriptor, + LPCSTR fileName, + LPCSTR filePathForError, + const SharedMemoryId *id, + bool createIfNotExist = true, + bool *createdRef = nullptr); static void CloseFile(int fileDescriptor); - static int ChangeMode(LPCSTR path, mode_t mode); - static SIZE_T GetFileSize(SharedMemorySystemCallErrors *errors, LPCSTR filePath, int fileDescriptor); static void SetFileSize(SharedMemorySystemCallErrors *errors, LPCSTR filePath, int fileDescriptor, SIZE_T byteCount); @@ -150,17 +198,21 @@ class SharedMemoryId LPCSTR m_name; SIZE_T m_nameCharCount; bool m_isSessionScope; // false indicates global scope + bool m_isUserScope; + uid_t m_userScopeUid; public: SharedMemoryId(); - SharedMemoryId(LPCSTR name, SIZE_T nameCharCount, bool isSessionScope); - SharedMemoryId(LPCSTR name); + SharedMemoryId(LPCSTR name, bool isUserScope); public: LPCSTR GetName() const; SIZE_T GetNameCharCount() const; + void ReplaceNamePtr(LPCSTR name); bool IsSessionScope() const; - bool Equals(SharedMemoryId *other) const; + bool IsUserScope() const; + uid_t GetUserScopeUid() const; + bool Equals(const SharedMemoryId *other) const; public: bool AppendSessionDirectoryName(PathCharString& path) const; @@ -222,7 +274,7 @@ class SharedMemoryProcessDataHeader SharedMemoryProcessDataHeader *m_nextInProcessDataHeaderList; public: - static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, SharedMemorySharedDataHeader requiredSharedDataHeader, SIZE_T sharedDataByteCount, bool createIfNotExist, bool *createdRef); + static SharedMemoryProcessDataHeader *CreateOrOpen(SharedMemorySystemCallErrors *errors, LPCSTR name, bool isUserScope, SharedMemorySharedDataHeader requiredSharedDataHeader, SIZE_T sharedDataByteCount, bool createIfNotExist, bool *createdRef); public: static SharedMemoryProcessDataHeader *PalObject_GetProcessDataHeader(CorUnix::IPalObject *object); @@ -230,14 +282,14 @@ class SharedMemoryProcessDataHeader static void PalObject_Close(CorUnix::CPalThread *thread, CorUnix::IPalObject *object, bool isShuttingDown, bool cleanUpPalSharedState); private: - SharedMemoryProcessDataHeader(SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount); + SharedMemoryProcessDataHeader(const SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount); public: - static SharedMemoryProcessDataHeader *New(SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount); + static SharedMemoryProcessDataHeader *New(const SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount); ~SharedMemoryProcessDataHeader(); void Close(); public: - SharedMemoryId *GetId(); + const SharedMemoryId *GetId() const; SharedMemoryProcessDataBase *GetData() const; void SetData(SharedMemoryProcessDataBase *data); SharedMemorySharedDataHeader *GetSharedDataHeader() const; @@ -254,10 +306,46 @@ class SharedMemoryManager { private: static CRITICAL_SECTION s_creationDeletionProcessLock; + static int s_runtimeDirectoryFileDescriptor; static int s_creationDeletionLockFileDescriptor; - static PathCharString* s_runtimeTempDirectoryPath; - static PathCharString* s_sharedMemoryDirectoryPath; + struct UserScopeUidData + { + uid_t userScopeUid; + LPCSTR sharedFilesPath; + SIZE_T persistedUserHomeDirectoryPathCharCount; + bool isApplicationContainer; + int runtimeDirectoryFileDescriptor; + int sharedMemoryDirectoryFileDescriptor; + + UserScopeUidData() + : userScopeUid((uid_t)0), + sharedFilesPath(nullptr), + persistedUserHomeDirectoryPathCharCount(0), + isApplicationContainer(false), + runtimeDirectoryFileDescriptor(-1), + sharedMemoryDirectoryFileDescriptor(-1) + { + } + + UserScopeUidData( + uid_t userScopeUid, + LPCSTR sharedFilesPath, + SIZE_T persistedUserHomeDirectoryPathCharCount, + bool isApplicationContainer) + : userScopeUid(userScopeUid), + sharedFilesPath(sharedFilesPath), + persistedUserHomeDirectoryPathCharCount(persistedUserHomeDirectoryPathCharCount), + isApplicationContainer(isApplicationContainer), + runtimeDirectoryFileDescriptor(-1), + sharedMemoryDirectoryFileDescriptor(-1) + { + } + }; + + static UserScopeUidData *s_userScopeUidData; + static int s_userScopeUidDataCount; + static int s_userScopeUidDataCapacity; private: static SharedMemoryProcessDataHeader *s_processDataHeaderListHead; @@ -269,17 +357,26 @@ class SharedMemoryManager #endif // _DEBUG public: - static bool StaticInitialize(); + static void StaticInitialize(); static void StaticClose(); public: static void AcquireCreationDeletionProcessLock(); static void ReleaseCreationDeletionProcessLock(); - static void AcquireCreationDeletionFileLock(SharedMemorySystemCallErrors *errors); - static void ReleaseCreationDeletionFileLock(); - -public: - static bool CopySharedMemoryBasePath(PathCharString& destination); + static void AcquireCreationDeletionFileLock(SharedMemorySystemCallErrors *errors, const SharedMemoryId *id); + static void ReleaseCreationDeletionFileLock(const SharedMemoryId *id); + static int GetRuntimeDirectoryFileDescriptor(const SharedMemoryId *id); + static int GetSharedMemoryDirectoryFileDescriptor(const SharedMemoryId *id); + static LPCSTR GetUserScopeSharedFilesPath(uid_t userScopeUid); + static SIZE_T GetUserScopePersistedHomeDirectoryPathCharCount(uid_t userScopeUid); + static LPCSTR GetRuntimeTempDirectoryName(const SharedMemoryId *id); +private: + static UserScopeUidData *GetOrCreateUserScopeUidData(uid_t userScopeUid); + static UserScopeUidData *FindUserScopeUidData(uid_t userScopeUid); + static void SetUserScopeUidDirectoryFileDescriptors( + uid_t userScopeUid, + int runtimeDirectoryFileDescriptor, + int sharedMemoryDirectoryFileDescriptor); #ifdef _DEBUG public: @@ -290,7 +387,7 @@ class SharedMemoryManager public: static void AddProcessDataHeader(SharedMemoryProcessDataHeader *processDataHeader); static void RemoveProcessDataHeader(SharedMemoryProcessDataHeader *processDataHeader); - static SharedMemoryProcessDataHeader *FindProcessDataHeader(SharedMemoryId *id); + static SharedMemoryProcessDataHeader *FindProcessDataHeader(const SharedMemoryId *id); }; #endif // !_PAL_SHARED_MEMORY_H_ diff --git a/src/coreclr/pal/src/include/pal/sharedmemory.inl b/src/coreclr/pal/src/include/pal/sharedmemory.inl deleted file mode 100644 index 598172c2bbc72b..00000000000000 --- a/src/coreclr/pal/src/include/pal/sharedmemory.inl +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#ifndef _PAL_SHARED_MEMORY_INL_ -#define _PAL_SHARED_MEMORY_INL_ - -#include "sharedmemory.h" - -#include "dbgmsg.h" - -#include - -template -void SharedMemoryHelpers::BuildSharedFilesPath( - PathCharString& destination, - const char (&suffix)[SuffixByteCount]) -{ - BuildSharedFilesPath(destination, suffix, SuffixByteCount - 1); -} - -#endif // !_PAL_SHARED_MEMORY_INL_ diff --git a/src/coreclr/pal/src/init/pal.cpp b/src/coreclr/pal/src/init/pal.cpp index 63ee86e291ff75..e6a2dded358254 100644 --- a/src/coreclr/pal/src/init/pal.cpp +++ b/src/coreclr/pal/src/init/pal.cpp @@ -435,12 +435,7 @@ Initialize( // we use large numbers of threads or have many open files. } - if (!SharedMemoryManager::StaticInitialize()) - { - ERROR("Shared memory static initialization failed!\n"); - palError = ERROR_PALINIT_SHARED_MEMORY_MANAGER; - goto CLEANUP0; - } + SharedMemoryManager::StaticInitialize(); /* initialize the shared memory infrastructure */ if (!SHMInitialize()) diff --git a/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp b/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp index ba9447b889c39c..16c8df78efd874 100644 --- a/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp +++ b/src/coreclr/pal/src/sharedmemory/sharedmemory.cpp @@ -19,6 +19,7 @@ SET_DEFAULT_DEBUG_CHANNEL(SHMEM); // some headers have code with asserts, so do #include #include +#include #include #include #include @@ -26,8 +27,6 @@ SET_DEFAULT_DEBUG_CHANNEL(SHMEM); // some headers have code with asserts, so do using namespace CorUnix; -#include "pal/sharedmemory.inl" - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // AutoFreeBuffer @@ -128,11 +127,15 @@ void SharedMemorySystemCallErrors::Append(LPCSTR format, ...) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // SharedMemoryHelpers -const mode_t SharedMemoryHelpers::PermissionsMask_CurrentUser_ReadWriteExecute = S_IRUSR | S_IWUSR | S_IXUSR; +const mode_t SharedMemoryHelpers::PermissionsMask_OwnerUser_ReadWrite = S_IRUSR | S_IWUSR; +const mode_t SharedMemoryHelpers::PermissionsMask_OwnerUser_ReadWriteExecute = S_IRUSR | S_IWUSR | S_IXUSR; +const mode_t SharedMemoryHelpers::PermissionsMask_NonOwnerUsers_Write = S_IWGRP | S_IWOTH; const mode_t SharedMemoryHelpers::PermissionsMask_AllUsers_ReadWrite = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; const mode_t SharedMemoryHelpers::PermissionsMask_AllUsers_ReadWriteExecute = PermissionsMask_AllUsers_ReadWrite | (S_IXUSR | S_IXGRP | S_IXOTH); +const mode_t SharedMemoryHelpers::PermissionsMask_Sticky = S_ISVTX; +const int SharedMemoryHelpers::TransientRetryCount = 10; const UINT32 SharedMemoryHelpers::InvalidProcessId = static_cast(-1); const SIZE_T SharedMemoryHelpers::InvalidThreadId = static_cast(-1); const UINT64 SharedMemoryHelpers::InvalidSharedThreadId = static_cast(-1); @@ -159,204 +162,327 @@ SIZE_T SharedMemoryHelpers::AlignUp(SIZE_T value, SIZE_T alignment) return AlignDown(value + (alignment - 1), alignment); } -bool SharedMemoryHelpers::EnsureDirectoryExists( +bool SharedMemoryHelpers::AppendPathComponent(PathCharString& path, LPCSTR component) +{ + _ASSERTE(component != nullptr); + _ASSERTE(component[0] != '\0'); + + SIZE_T pathCharCount = path.GetCount(); + return + (pathCharCount == 0 || path.GetString()[pathCharCount - 1] == '/' || path.Append('/')) && + path.Append(component, strlen(component)); +} + +int SharedMemoryHelpers::OpenSharedFilesSystemDirectory( SharedMemorySystemCallErrors *errors, - const char *path, - bool isGlobalLockAcquired, - bool createIfNotExist, - bool isSystemDirectory) + const SharedMemoryId *id) { - _ASSERTE(path != nullptr); - _ASSERTE(!(isSystemDirectory && createIfNotExist)); // should not create or change permissions on system directories + _ASSERTE(id != nullptr); _ASSERTE(SharedMemoryManager::IsCreationDeletionProcessLockAcquired()); - _ASSERTE(!isGlobalLockAcquired || SharedMemoryManager::IsCreationDeletionFileLockAcquired()); - // Check if the path already exists - struct stat statInfo; - int statResult = stat(path, &statInfo); - if (statResult != 0 && errno == ENOENT) + PathCharString path; + BuildSharedFilesSystemPath(path, id); + int transientRetryCount = 0; + + while (true) { - if (!createIfNotExist) + int directoryFileDescriptor; + do { - return false; - } - - // The path does not exist, create the directory. The permissions mask passed to mkdir() is filtered by the process' - // permissions umask, so mkdir() may not set all of the requested permissions. We need to use chmod() to set the proper - // permissions. That creates a race when there is no global lock acquired when creating the directory. Another user's - // process may create the directory and this user's process may try to use it before the other process sets the full - // permissions. In that case, create a temporary directory first, set the permissions, and rename it to the actual - // directory name. + directoryFileDescriptor = InternalOpen(path, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC, 0); + } while (directoryFileDescriptor == -1 && errno == EINTR); - if (isGlobalLockAcquired) + if (directoryFileDescriptor != -1) { - int operationResult = mkdir(path, PermissionsMask_AllUsers_ReadWriteExecute); - if (operationResult != 0) + struct stat statInfo; + int statResult; + do { - if (errors != nullptr) - { - int errorCode = errno; - errors->Append( - "mkdir(\"%s\", AllUsers_ReadWriteExecute) == %d; errno == %s;", - path, - operationResult, - GetFriendlyErrorCodeString(errorCode)); - } + statResult = fstat(directoryFileDescriptor, &statInfo); + } while (statResult != 0 && errno == EINTR); - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); - } - - operationResult = ChangeMode(path, PermissionsMask_AllUsers_ReadWriteExecute); - if (operationResult != 0) + if (statResult == 0) { - if (errors != nullptr) + // For system directories (such as TEMP_DIRECTORY_PATH), require sufficient permissions only for the + // owner user. For instance, "docker run --mount ..." to mount /tmp to some directory on the host mounts the + // destination directory with the same permissions as the source directory, which may not include some + // permissions for other users. In the docker container, other user permissions are typically not relevant and + // relaxing the permissions requirement allows for that scenario to work without having to work around it by + // first giving sufficient permissions for all users. + // + // If the directory is being used for user-scoped shared memory data, also ensure that either it has the sticky + // bit or it's owned by the current user and without write access for other users. + mode_t ownerPermissions = statInfo.st_mode & PermissionsMask_OwnerUser_ReadWriteExecute; + bool safeForUserScope = + !id->IsUserScope() || + (statInfo.st_mode & PermissionsMask_Sticky) != 0 || + (statInfo.st_uid == id->GetUserScopeUid() && + (statInfo.st_mode & PermissionsMask_NonOwnerUsers_Write) == 0); + if (ownerPermissions == PermissionsMask_OwnerUser_ReadWriteExecute && safeForUserScope) { - int errorCode = errno; - errors->Append( - "chmod(\"%s\", AllUsers_ReadWriteExecute) == %d; errno == %s;", - path, - operationResult, - GetFriendlyErrorCodeString(errorCode)); + return directoryFileDescriptor; } - - rmdir(path); - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } - return true; - } - - PathCharString tempPath; - BuildSharedFilesPath(tempPath, SHARED_MEMORY_UNIQUE_TEMP_NAME_TEMPLATE); - - if (mkdtemp(tempPath.OpenStringBuffer()) == nullptr) - { - if (errors != nullptr) + int errorCode = errno; + CloseFile(directoryFileDescriptor); + if (statResult != 0 && (errorCode == ENOENT || errorCode == EACCES) && + transientRetryCount++ < TransientRetryCount) { - int errorCode = errno; - errors->Append( - "mkdtemp(\"%s\") == nullptr; errno == %s;", - (const char *)tempPath, - GetFriendlyErrorCodeString(errorCode)); + usleep(1000); + continue; } - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); - } + if (statResult == 0 && transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } - int operationResult = ChangeMode(tempPath, PermissionsMask_AllUsers_ReadWriteExecute); - if (operationResult != 0) - { if (errors != nullptr) { - int errorCode = errno; - errors->Append( - "chmod(\"%s\", AllUsers_ReadWriteExecute) == %d; errno == %s;", - (const char *)tempPath, - operationResult, - GetFriendlyErrorCodeString(errorCode)); + if (statResult != 0) + { + errors->Append( + "fstat(directoryFileDescriptor for \"%s\", ...) == %d; errno == %s;", + (const char *)path, + statResult, + GetFriendlyErrorCodeString(errorCode)); + } + else + { + errors->Append( + "fstat(directoryFileDescriptor for \"%s\", &info) == 0; info.st_mode == 0x%x; info.st_uid == %u;", + (const char *)path, + (int)statInfo.st_mode, + (unsigned int)statInfo.st_uid); + } } - - rmdir(tempPath); throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } - if (rename(tempPath, path) == 0) + int errorCode = errno; + if ((errorCode == ENOENT || errorCode == EACCES) && transientRetryCount++ < TransientRetryCount) { - return true; + usleep(1000); + continue; } - // Another process may have beaten us to it. Delete the temp directory and continue to check the requested directory to - // see if it meets our needs. - rmdir(tempPath); - statResult = stat(path, &statInfo); + if (errors != nullptr) + { + errors->Append( + "open(\"%s\", O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC, 0) == -1; errno == %s;", + (const char *)path, + GetFriendlyErrorCodeString(errorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } +} - // If the path exists, check that it's a directory - if (statResult != 0 || !(statInfo.st_mode & S_IFDIR)) +int SharedMemoryHelpers::EnsureDirectoryExists( + SharedMemorySystemCallErrors *errors, + int parentDirectoryFileDescriptor, + LPCSTR directoryName, + LPCSTR directoryPathForError, + const SharedMemoryId *id, + bool createIfNotExist) +{ + _ASSERTE(parentDirectoryFileDescriptor != -1); + _ASSERTE(directoryName != nullptr); + _ASSERTE(directoryName[0] != '\0'); + _ASSERTE(directoryPathForError != nullptr); + _ASSERTE(id != nullptr); + _ASSERTE(SharedMemoryManager::IsCreationDeletionProcessLockAcquired()); + + mode_t permissionsMask = + id->IsUserScope() ? PermissionsMask_OwnerUser_ReadWriteExecute : PermissionsMask_AllUsers_ReadWriteExecute; + int transientRetryCount = 0; + + while (true) { - if (errors != nullptr) + int directoryFileDescriptor; + do + { + directoryFileDescriptor = + openat(parentDirectoryFileDescriptor, directoryName, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC); + } while (directoryFileDescriptor == -1 && errno == EINTR); + + bool created = false; + if (directoryFileDescriptor == -1) { - if (statResult != 0) + int openErrorCode = errno; + if (openErrorCode == ENOENT) { - int errorCode = errno; - errors->Append( - "stat(\"%s\", ...) == %d; errno == %s;", - path, - statResult, - GetFriendlyErrorCodeString(errorCode)); + if (!createIfNotExist) + { + return -1; + } + + int mkdirResult; + do + { + mkdirResult = mkdirat(parentDirectoryFileDescriptor, directoryName, permissionsMask); + } while (mkdirResult != 0 && errno == EINTR); + + if (mkdirResult == 0) + { + created = true; + do + { + directoryFileDescriptor = + openat(parentDirectoryFileDescriptor, directoryName, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC); + } while (directoryFileDescriptor == -1 && errno == EINTR); + + if (directoryFileDescriptor == -1) + { + int errorCode = errno; + unlinkat(parentDirectoryFileDescriptor, directoryName, AT_REMOVEDIR); + if (errors != nullptr) + { + errors->Append( + "openat(%d, \"%s\", O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC) == -1; errno == %s;", + parentDirectoryFileDescriptor, + directoryName, + GetFriendlyErrorCodeString(errorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + + int chmodResult; + do + { + chmodResult = fchmod(directoryFileDescriptor, permissionsMask); + } while (chmodResult != 0 && errno == EINTR); + + if (chmodResult != 0) + { + int errorCode = errno; + CloseFile(directoryFileDescriptor); + unlinkat(parentDirectoryFileDescriptor, directoryName, AT_REMOVEDIR); + if (errors != nullptr) + { + errors->Append( + "fchmod(directoryFileDescriptor for \"%s\", 0x%x) == %d; errno == %s;", + directoryPathForError, + (int)permissionsMask, + chmodResult, + GetFriendlyErrorCodeString(errorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + } + else + { + int mkdirErrorCode = errno; + if (mkdirErrorCode == EEXIST && transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } + + if (errors != nullptr) + { + errors->Append( + "mkdirat(%d, \"%s\", 0x%x) == -1; errno == %s;", + parentDirectoryFileDescriptor, + directoryName, + (int)permissionsMask, + GetFriendlyErrorCodeString(mkdirErrorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } } else { - errors->Append( - "stat(\"%s\", &info) == 0; info.st_mode == 0x%x; (info.st_mode & 0x%x) == 0;", - path, - (int)statInfo.st_mode, - (int)S_IFDIR); + if ((openErrorCode == ENOENT || openErrorCode == EACCES) && + transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } + + if (errors != nullptr) + { + errors->Append( + "openat(%d, \"%s\", O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC) == -1; errno == %s;", + parentDirectoryFileDescriptor, + directoryName, + GetFriendlyErrorCodeString(openErrorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } } - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + return directoryFileDescriptor; } +} - if (isSystemDirectory) +int SharedMemoryHelpers::Open(SharedMemorySystemCallErrors *errors, LPCSTR path, int flags, mode_t mode) +{ + int openErrorCode; + + flags |= O_CLOEXEC; + do { - // For system directories (such as TEMP_DIRECTORY_PATH), require sufficient permissions only for the - // current user. For instance, "docker run --mount ..." to mount /tmp to some directory on the host mounts the - // destination directory with the same permissions as the source directory, which may not include some permissions for - // other users. In the docker container, other user permissions are typically not relevant and relaxing the permissions - // requirement allows for that scenario to work without having to work around it by first giving sufficient permissions - // for all users. - if ((statInfo.st_mode & PermissionsMask_CurrentUser_ReadWriteExecute) == PermissionsMask_CurrentUser_ReadWriteExecute) + int fileDescriptor = InternalOpen(path, flags, mode); + if (fileDescriptor != -1) { - return true; + return fileDescriptor; } + openErrorCode = errno; + } while (openErrorCode == EINTR); - if (errors != nullptr) - { - errors->Append( - "stat(\"%s\", &info) == 0; info.st_mode == 0x%x; (info.st_mode & CurrentUser_ReadWriteExecute) != CurrentUser_ReadWriteExecute;", - path, - (int)statInfo.st_mode); - } + SharedMemoryError sharedMemoryError; + switch (openErrorCode) + { + case ENOENT: + _ASSERTE(!(flags & O_CREAT)); + errno = openErrorCode; + return -1; - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + case ENAMETOOLONG: + sharedMemoryError = SharedMemoryError::NameTooLong; + break; + + case EMFILE: + case ENFILE: + case ENOMEM: + sharedMemoryError = SharedMemoryError::OutOfMemory; + break; + + default: + sharedMemoryError = SharedMemoryError::IO; + break; } - // For non-system directories (such as gSharedFilesPath/SHARED_MEMORY_RUNTIME_TEMP_DIRECTORY_NAME), - // require sufficient permissions for all users and try to update them if requested to create the directory, so that - // shared memory files may be shared by all processes on the system. - if ((statInfo.st_mode & PermissionsMask_AllUsers_ReadWriteExecute) == PermissionsMask_AllUsers_ReadWriteExecute) + if (sharedMemoryError != SharedMemoryError::NameTooLong && errors != nullptr) { - return true; + errors->Append( + "open(\"%s\", 0x%x, 0x%x) == -1; errno == %s;", + path, + flags, + (int)mode, + GetFriendlyErrorCodeString(openErrorCode)); } - if (!createIfNotExist || ChangeMode(path, PermissionsMask_AllUsers_ReadWriteExecute) != 0) - { - // We were not asked to create the path or we weren't able to set the new permissions. - // As a last resort, check that at least the current user has full access. - if ((statInfo.st_mode & PermissionsMask_CurrentUser_ReadWriteExecute) != PermissionsMask_CurrentUser_ReadWriteExecute) - { - if (errors != nullptr) - { - errors->Append( - "stat(\"%s\", &info) == 0; info.st_mode == 0x%x; (info.st_mode & CurrentUser_ReadWriteExecute) != CurrentUser_ReadWriteExecute;", - path, - (int)statInfo.st_mode); - } - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); - } - } - return true; + throw SharedMemoryException(static_cast(sharedMemoryError)); } -int SharedMemoryHelpers::Open(SharedMemorySystemCallErrors *errors, LPCSTR path, int flags, mode_t mode) +int SharedMemoryHelpers::OpenAt( + SharedMemorySystemCallErrors *errors, + int directoryFileDescriptor, + LPCSTR path, + int flags, + mode_t mode) { - int openErrorCode; + _ASSERTE(directoryFileDescriptor != -1); + _ASSERTE(path != nullptr); + int openErrorCode; flags |= O_CLOEXEC; do { - int fileDescriptor = InternalOpen(path, flags, mode); + int fileDescriptor = openat(directoryFileDescriptor, path, flags, mode); if (fileDescriptor != -1) { return fileDescriptor; @@ -390,7 +516,8 @@ int SharedMemoryHelpers::Open(SharedMemorySystemCallErrors *errors, LPCSTR path, if (sharedMemoryError != SharedMemoryError::NameTooLong && errors != nullptr) { errors->Append( - "open(\"%s\", 0x%x, 0x%x) == -1; errno == %s;", + "openat(%d, \"%s\", 0x%x, 0x%x) == -1; errno == %s;", + directoryFileDescriptor, path, flags, (int)mode, @@ -405,73 +532,157 @@ int SharedMemoryHelpers::OpenDirectory(SharedMemorySystemCallErrors *errors, LPC _ASSERTE(path != nullptr); _ASSERTE(path[0] != '\0'); - int fileDescriptor = Open(errors, path, O_RDONLY); + int fileDescriptor = Open(errors, path, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC); _ASSERTE(fileDescriptor != -1 || errno == ENOENT); return fileDescriptor; } int SharedMemoryHelpers::CreateOrOpenFile( SharedMemorySystemCallErrors *errors, - LPCSTR path, + int directoryFileDescriptor, + LPCSTR fileName, + LPCSTR filePathForError, + const SharedMemoryId *id, bool createIfNotExist, bool *createdRef) { - _ASSERTE(path != nullptr); - _ASSERTE(path[0] != '\0'); + _ASSERTE(directoryFileDescriptor != -1); + _ASSERTE(fileName != nullptr); + _ASSERTE(fileName[0] != '\0'); + _ASSERTE(filePathForError != nullptr); + _ASSERTE(filePathForError[0] != '\0'); + _ASSERTE(id != nullptr); _ASSERTE(SharedMemoryManager::IsCreationDeletionProcessLockAcquired()); _ASSERTE(!createIfNotExist || SharedMemoryManager::IsCreationDeletionFileLockAcquired()); - // Try to open the file - int openFlags = O_RDWR; - int fileDescriptor = Open(errors, path, openFlags); - if (fileDescriptor != -1) + int transientRetryCount = 0; + while (true) { - if (createdRef != nullptr) + int openFlags = O_RDWR | O_NOFOLLOW | O_CLOEXEC; + int fileDescriptor = -1; + try { - *createdRef = false; + fileDescriptor = OpenAt(errors, directoryFileDescriptor, fileName, openFlags); } - return fileDescriptor; - } - _ASSERTE(errno == ENOENT); - if (!createIfNotExist) - { - if (createdRef != nullptr) + catch (SharedMemoryException) { - *createdRef = false; + if (errno == EACCES && transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } + throw; } - return -1; - } - // File does not exist, create the file - openFlags |= O_CREAT | O_EXCL; - fileDescriptor = Open(errors, path, openFlags, PermissionsMask_AllUsers_ReadWrite); - _ASSERTE(fileDescriptor != -1); + if (fileDescriptor != -1) + { + if (id->IsUserScope()) + { + struct stat statInfo; + int statResult = fstat(fileDescriptor, &statInfo); + int errorCode = errno; + if (statResult != 0) + { + if (errors != nullptr) + { + errors->Append( + "fstat(\"%s\", ...) == %d; errno == %s;", + filePathForError, + statResult, + GetFriendlyErrorCodeString(errorCode)); + } + CloseFile(fileDescriptor); + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } - // The permissions mask passed to open() is filtered by the process' permissions umask, so open() may not set all of - // the requested permissions. Use chmod() to set the proper permissions. - int operationResult = ChangeMode(path, PermissionsMask_AllUsers_ReadWrite); - if (operationResult != 0) - { - if (errors != nullptr) + if (statInfo.st_uid != id->GetUserScopeUid() || + (statInfo.st_mode & PermissionsMask_AllUsers_ReadWriteExecute) != PermissionsMask_OwnerUser_ReadWrite) + { + CloseFile(fileDescriptor); + if (transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } + + if (errors != nullptr) + { + errors->Append( + "fstat(\"%s\", &info) == 0; info.st_mode == 0x%x; info.st_uid == %u; expected mode == 0x%x, uid == %u;", + filePathForError, + (int)statInfo.st_mode, + (unsigned int)statInfo.st_uid, + (int)PermissionsMask_OwnerUser_ReadWrite, + (unsigned int)id->GetUserScopeUid()); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + } + + if (createdRef != nullptr) + { + *createdRef = false; + } + return fileDescriptor; + } + + _ASSERTE(errno == ENOENT); + if (!createIfNotExist) { - int errorCode = errno; - errors->Append( - "chmod(\"%s\", AllUsers_ReadWrite) == %d; errno == %s;", - path, - operationResult, - GetFriendlyErrorCodeString(errorCode)); + if (createdRef != nullptr) + { + *createdRef = false; + } + return -1; } - CloseFile(fileDescriptor); - unlink(path); - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); - } + openFlags |= O_CREAT | O_EXCL | O_CLOEXEC; + mode_t permissionsMask = + id->IsUserScope() ? PermissionsMask_OwnerUser_ReadWrite : PermissionsMask_AllUsers_ReadWrite; + try + { + fileDescriptor = OpenAt(errors, directoryFileDescriptor, fileName, openFlags, permissionsMask); + } + catch (SharedMemoryException) + { + if (errno == EEXIST && transientRetryCount++ < TransientRetryCount) + { + continue; + } + throw; + } + _ASSERTE(fileDescriptor != -1); - if (createdRef != nullptr) - { - *createdRef = true; + // The creation mode is filtered by umask. Set the exact mode through the descriptor so a replaced path is not followed. + int operationResult; + do + { + operationResult = fchmod(fileDescriptor, permissionsMask); + } while (operationResult != 0 && errno == EINTR); + + if (operationResult != 0) + { + int errorCode = errno; + if (errors != nullptr) + { + errors->Append( + "fchmod(fileDescriptor, 0x%x) == %d; errno == %s;", + (int)permissionsMask, + operationResult, + GetFriendlyErrorCodeString(errorCode)); + } + + CloseFile(fileDescriptor); + unlinkat(directoryFileDescriptor, fileName, 0); + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + + if (createdRef != nullptr) + { + *createdRef = true; + } + return fileDescriptor; } - return fileDescriptor; } void SharedMemoryHelpers::CloseFile(int fileDescriptor) @@ -485,20 +696,6 @@ void SharedMemoryHelpers::CloseFile(int fileDescriptor) } while (closeResult != 0 && errno == EINTR); } -int SharedMemoryHelpers::ChangeMode(LPCSTR path, mode_t mode) -{ - _ASSERTE(path != nullptr); - _ASSERTE(path[0] != '\0'); - - int chmodResult; - do - { - chmodResult = chmod(path, mode); - } while (chmodResult != 0 && errno == EINTR); - - return chmodResult; -} - SIZE_T SharedMemoryHelpers::GetFileSize(SharedMemorySystemCallErrors *errors, LPCSTR filePath, int fileDescriptor) { _ASSERTE(filePath != nullptr); @@ -556,120 +753,468 @@ void SharedMemoryHelpers::SetFileSize( ftruncateResult, GetFriendlyErrorCodeString(errorCode)); } - + + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + } +} + +void *SharedMemoryHelpers::MemoryMapFile( + SharedMemorySystemCallErrors *errors, + LPCSTR filePath, + int fileDescriptor, + SIZE_T byteCount) +{ + _ASSERTE(filePath != nullptr); + _ASSERTE(filePath[0] != '\0'); + _ASSERTE(fileDescriptor != -1); + _ASSERTE(byteCount > sizeof(SharedMemorySharedDataHeader)); + _ASSERTE(AlignDown(byteCount, GetVirtualPageSize()) == byteCount); + + void *sharedMemoryBuffer = mmap(nullptr, byteCount, PROT_READ | PROT_WRITE, MAP_SHARED, fileDescriptor, 0); + if (sharedMemoryBuffer != MAP_FAILED) + { + return sharedMemoryBuffer; + } + + int errorCode = errno; + SharedMemoryError sharedMemoryError; + switch (errorCode) + { + case EMFILE: + case ENFILE: + case ENOMEM: + sharedMemoryError = SharedMemoryError::OutOfMemory; + break; + + default: + sharedMemoryError = SharedMemoryError::IO; + break; + } + + if (errors != nullptr) + { + errors->Append( + "mmap(nullptr, %zu, PROT_READ | PROT_WRITE, MAP_SHARED, \"%s\", 0) == MAP_FAILED; errno == %s;", + byteCount, + filePath, + GetFriendlyErrorCodeString(errorCode)); + } + + throw SharedMemoryException(static_cast(sharedMemoryError)); +} + +bool SharedMemoryHelpers::TryAcquireFileLock(SharedMemorySystemCallErrors *errors, int fileDescriptor, int operation) +{ + // A file lock is acquired once per file descriptor, so the caller will need to synchronize threads of this process + + _ASSERTE(fileDescriptor != -1); + _ASSERTE((operation & LOCK_EX) ^ (operation & LOCK_SH)); + _ASSERTE(!(operation & LOCK_UN)); + + while (true) + { + int flockResult = flock(fileDescriptor, operation); + if (flockResult == 0) + { + return true; + } + + int flockError = errno; + SharedMemoryError sharedMemoryError = SharedMemoryError::IO; + switch (flockError) + { + case EWOULDBLOCK: + return false; + + case EINTR: + continue; + + case ENOLCK: + sharedMemoryError = SharedMemoryError::OutOfMemory; + break; + } + + if (errors != nullptr) + { + errors->Append( + "flock(%d, %s%s) == %d; errno == %s;", + fileDescriptor, + operation & LOCK_EX ? "LOCK_EX" : "LOCK_SH", + operation & LOCK_NB ? " | LOCK_NB" : "", + flockResult, + GetFriendlyErrorCodeString(flockError)); + } + + throw SharedMemoryException(static_cast(sharedMemoryError)); + } +} + +void SharedMemoryHelpers::ReleaseFileLock(int fileDescriptor) +{ + _ASSERTE(fileDescriptor != -1); + + int flockResult; + do + { + flockResult = flock(fileDescriptor, LOCK_UN); + } while (flockResult != 0 && errno == EINTR); +} + +void SharedMemoryHelpers::BuildUserSpecificSharedFilesPath( + PathCharString& destination, + uid_t uid, + SIZE_T *persistedUserHomeDirectoryPathCharCountRef, + bool *isApplicationContainerRef) +{ + _ASSERTE(persistedUserHomeDirectoryPathCharCountRef != nullptr); + _ASSERTE(isApplicationContainerRef != nullptr); + *persistedUserHomeDirectoryPathCharCountRef = 0; + *isApplicationContainerRef = false; + +#ifdef __APPLE__ + const char *sharedMemoryApplicationGroupId = PAL_getenv("DOTNET_SHARED_MEMORY_APPLICATION_GROUP_ID"); + if (sharedMemoryApplicationGroupId != nullptr && sharedMemoryApplicationGroupId[0] != '\0') + { + SIZE_T applicationGroupIdLength = strlen(sharedMemoryApplicationGroupId); + if (applicationGroupIdLength > MAX_APPLICATION_GROUP_ID_LENGTH || + !GetApplicationContainerFolder( + destination, + sharedMemoryApplicationGroupId, + static_cast(applicationGroupIdLength))) + { + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + *isApplicationContainerRef = true; + return; + } + + if (gApplicationGroupId != nullptr && gApplicationGroupId[0] != '\0') + { + VerifyStringOperation(destination.Set(*gSharedFilesPath)); + *isApplicationContainerRef = true; + return; + } + + const char *tmpDirectory = PAL_getenv("TMPDIR"); + if (tmpDirectory != nullptr && tmpDirectory[0] != '\0') + { + VerifyStringOperation(destination.Set(tmpDirectory, strlen(tmpDirectory))); + return; + } +#elif defined(__linux__) + const char *xdgRuntimeDirectory = PAL_getenv("XDG_RUNTIME_DIR"); + if (xdgRuntimeDirectory != nullptr && xdgRuntimeDirectory[0] != '\0') + { + VerifyStringOperation(destination.Set(xdgRuntimeDirectory, strlen(xdgRuntimeDirectory))); + return; + } +#endif + + const char *homeDirectory = uid == geteuid() ? PAL_getenv("HOME") : nullptr; + if (homeDirectory != nullptr && homeDirectory[0] != '\0') + { + SIZE_T homeDirectoryLength = strlen(homeDirectory); + VerifyStringOperation(destination.Set(homeDirectory, homeDirectoryLength)); + while (homeDirectoryLength > 1 && homeDirectory[homeDirectoryLength - 1] == '/') + { + destination.CloseBuffer(--homeDirectoryLength); + } + *persistedUserHomeDirectoryPathCharCountRef = homeDirectoryLength; + if (homeDirectory[homeDirectoryLength - 1] != '/') + { + VerifyStringOperation(destination.Append('/')); + } + VerifyStringOperation(destination.Append(SHARED_MEMORY_PERSISTED_USER_DIRECTORY_SUFFIX)); + return; + } + + SIZE_T bufferSize = 1024; + while (true) + { + char *buffer = reinterpret_cast(Alloc(bufferSize)); + AutoFreeBuffer autoFreeBuffer(buffer); + struct passwd pwd; + struct passwd *result = nullptr; + int error = getpwuid_r(uid, &pwd, buffer, bufferSize, &result); + if (error == 0) + { + if (result == nullptr || result->pw_dir == nullptr || result->pw_dir[0] == '\0') + { + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + + SIZE_T homeDirectoryLength = strlen(result->pw_dir); + VerifyStringOperation(destination.Set(result->pw_dir, homeDirectoryLength)); + while (homeDirectoryLength > 1 && result->pw_dir[homeDirectoryLength - 1] == '/') + { + destination.CloseBuffer(--homeDirectoryLength); + } + *persistedUserHomeDirectoryPathCharCountRef = homeDirectoryLength; + if (result->pw_dir[homeDirectoryLength - 1] != '/') + { + VerifyStringOperation(destination.Append('/')); + } + VerifyStringOperation(destination.Append(SHARED_MEMORY_PERSISTED_USER_DIRECTORY_SUFFIX)); + return; + } + + if (error != ERANGE || bufferSize > static_cast(-1) / 2) + { + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + bufferSize *= 2; + } +} + +void SharedMemoryHelpers::EnsurePersistedUserDirectoryExists( + SharedMemorySystemCallErrors *errors, + LPCSTR path, + SIZE_T homeDirectoryPathCharCount, + const SharedMemoryId *id) +{ + _ASSERTE(path != nullptr); + _ASSERTE(homeDirectoryPathCharCount != 0); + _ASSERTE(id != nullptr); + _ASSERTE(id->IsUserScope()); + _ASSERTE(SharedMemoryManager::IsCreationDeletionProcessLockAcquired()); + _ASSERTE(!SharedMemoryManager::IsCreationDeletionFileLockAcquired()); + + PathCharString homeDirectoryPath; + VerifyStringOperation(homeDirectoryPath.Set(path, homeDirectoryPathCharCount)); + + int homeDirectoryFileDescriptor = OpenDirectory(errors, homeDirectoryPath); + if (homeDirectoryFileDescriptor == -1) + { + if (errors != nullptr) + { + errors->Append("The persisted user home directory \"%s\" does not exist;", (const char *)homeDirectoryPath); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + + static const char *const directoryNames[] = + { + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_0, + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_1, + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_2, + SHARED_MEMORY_PERSISTED_USER_DIRECTORY_NAME_3 + }; + + struct AutoCleanup + { + int m_directoryFileDescriptors[ARRAY_SIZE(directoryNames) + 1]; + bool m_createdDirectories[ARRAY_SIZE(directoryNames)]; + int m_directoryCount; + bool m_cancel; + + AutoCleanup(int homeDirectoryFileDescriptor) + : m_directoryCount(0), m_cancel(false) + { + for (SIZE_T i = 0; i < ARRAY_SIZE(m_directoryFileDescriptors); i++) + { + m_directoryFileDescriptors[i] = -1; + } + for (SIZE_T i = 0; i < ARRAY_SIZE(m_createdDirectories); i++) + { + m_createdDirectories[i] = false; + } + m_directoryFileDescriptors[0] = homeDirectoryFileDescriptor; + } + + ~AutoCleanup() + { + for (int i = m_directoryCount; i >= 0; i--) + { + int fileDescriptor = m_directoryFileDescriptors[i]; + if (fileDescriptor != -1) + { + SharedMemoryHelpers::CloseFile(fileDescriptor); + } + + if (!m_cancel && i > 0 && m_createdDirectories[i - 1]) + { + unlinkat(m_directoryFileDescriptors[i - 1], directoryNames[i - 1], AT_REMOVEDIR); + } + } + } + } autoCleanup(homeDirectoryFileDescriptor); + + for (SIZE_T directoryIndex = 0; directoryIndex < ARRAY_SIZE(directoryNames); directoryIndex++) + { + int parentDirectoryFileDescriptor = autoCleanup.m_directoryFileDescriptors[directoryIndex]; + const char *directoryName = directoryNames[directoryIndex]; + bool created = false; + int directoryFileDescriptor = -1; + int transientRetryCount = 0; + + while (true) + { + do + { + directoryFileDescriptor = + openat(parentDirectoryFileDescriptor, directoryName, O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC); + } while (directoryFileDescriptor == -1 && errno == EINTR); + + if (directoryFileDescriptor != -1) + { + break; + } + + int openErrorCode = errno; + if (openErrorCode != ENOENT) + { + if (created) + { + unlinkat(parentDirectoryFileDescriptor, directoryName, AT_REMOVEDIR); + } + if (errors != nullptr) + { + errors->Append( + "openat(%d, \"%s\", O_RDONLY | O_NOFOLLOW | O_DIRECTORY | O_CLOEXEC) == -1; errno == %s;", + parentDirectoryFileDescriptor, + directoryName, + GetFriendlyErrorCodeString(openErrorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + + int mkdirResult; + do + { + mkdirResult = + mkdirat(parentDirectoryFileDescriptor, directoryName, PermissionsMask_OwnerUser_ReadWriteExecute); + } while (mkdirResult != 0 && errno == EINTR); + + if (mkdirResult == 0) + { + created = true; + continue; + } + + int mkdirErrorCode = errno; + if (mkdirErrorCode == EEXIST && transientRetryCount++ < TransientRetryCount) + { + usleep(1000); + continue; + } + + if (errors != nullptr) + { + errors->Append( + "mkdirat(%d, \"%s\", 0x%x) == -1; errno == %s;", + parentDirectoryFileDescriptor, + directoryName, + (int)PermissionsMask_OwnerUser_ReadWriteExecute, + GetFriendlyErrorCodeString(mkdirErrorCode)); + } + if (created) + { + unlinkat(parentDirectoryFileDescriptor, directoryName, AT_REMOVEDIR); + } throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } + + if (created) + { + int chmodResult; + do + { + chmodResult = fchmod(directoryFileDescriptor, PermissionsMask_OwnerUser_ReadWriteExecute); + } while (chmodResult != 0 && errno == EINTR); + + if (chmodResult != 0) + { + int chmodErrorCode = errno; + CloseFile(directoryFileDescriptor); + unlinkat(parentDirectoryFileDescriptor, directoryName, AT_REMOVEDIR); + if (errors != nullptr) + { + errors->Append( + "fchmod(directoryFileDescriptor for \"%s\", 0x%x) == %d; errno == %s;", + directoryName, + (int)PermissionsMask_OwnerUser_ReadWriteExecute, + chmodResult, + GetFriendlyErrorCodeString(chmodErrorCode)); + } + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } + } + + autoCleanup.m_directoryFileDescriptors[directoryIndex + 1] = directoryFileDescriptor; + autoCleanup.m_createdDirectories[directoryIndex] = created; + autoCleanup.m_directoryCount++; } + + autoCleanup.m_cancel = true; } -void *SharedMemoryHelpers::MemoryMapFile( - SharedMemorySystemCallErrors *errors, - LPCSTR filePath, - int fileDescriptor, - SIZE_T byteCount) +void SharedMemoryHelpers::BuildSharedFilesSystemPath(PathCharString& destination, const SharedMemoryId *id) { - _ASSERTE(filePath != nullptr); - _ASSERTE(filePath[0] != '\0'); - _ASSERTE(fileDescriptor != -1); - _ASSERTE(byteCount > sizeof(SharedMemorySharedDataHeader)); - _ASSERTE(AlignDown(byteCount, GetVirtualPageSize()) == byteCount); + _ASSERTE(id != nullptr); - void *sharedMemoryBuffer = mmap(nullptr, byteCount, PROT_READ | PROT_WRITE, MAP_SHARED, fileDescriptor, 0); - if (sharedMemoryBuffer != MAP_FAILED) + if (id->IsUserScope()) { - return sharedMemoryBuffer; + LPCSTR sharedFilesPath = SharedMemoryManager::GetUserScopeSharedFilesPath(id->GetUserScopeUid()); + VerifyStringOperation(destination.Set(sharedFilesPath, strlen(sharedFilesPath))); } - - int errorCode = errno; - SharedMemoryError sharedMemoryError; - switch (errorCode) + else { - case EMFILE: - case ENFILE: - case ENOMEM: - sharedMemoryError = SharedMemoryError::OutOfMemory; - break; - - default: - sharedMemoryError = SharedMemoryError::IO; - break; + VerifyStringOperation(destination.Set(*gSharedFilesPath)); } - if (errors != nullptr) + SIZE_T pathCharCount = destination.GetCount(); + _ASSERTE(pathCharCount != 0); + const char *path = destination; + while (pathCharCount > 1 && path[pathCharCount - 1] == '/') { - errors->Append( - "mmap(nullptr, %zu, PROT_READ | PROT_WRITE, MAP_SHARED, \"%s\", 0) == MAP_FAILED; errno == %s;", - byteCount, - filePath, - GetFriendlyErrorCodeString(errorCode)); + destination.CloseBuffer(--pathCharCount); + path = destination; } - - throw SharedMemoryException(static_cast(sharedMemoryError)); } -bool SharedMemoryHelpers::TryAcquireFileLock(SharedMemorySystemCallErrors *errors, int fileDescriptor, int operation) +void SharedMemoryHelpers::BuildRuntimeRootPath(PathCharString& destination, const SharedMemoryId *id) { - // A file lock is acquired once per file descriptor, so the caller will need to synchronize threads of this process - - _ASSERTE(fileDescriptor != -1); - _ASSERTE((operation & LOCK_EX) ^ (operation & LOCK_SH)); - _ASSERTE(!(operation & LOCK_UN)); - - while (true) - { - int flockResult = flock(fileDescriptor, operation); - if (flockResult == 0) - { - return true; - } - - int flockError = errno; - SharedMemoryError sharedMemoryError = SharedMemoryError::IO; - switch (flockError) - { - case EWOULDBLOCK: - return false; - - case EINTR: - continue; - - case ENOLCK: - sharedMemoryError = SharedMemoryError::OutOfMemory; - break; - } - - if (errors != nullptr) - { - errors->Append( - "flock(%d, %s%s) == %d; errno == %s;", - fileDescriptor, - operation & LOCK_EX ? "LOCK_EX" : "LOCK_SH", - operation & LOCK_NB ? " | LOCK_NB" : "", - flockResult, - GetFriendlyErrorCodeString(flockError)); - } + BuildSharedFilesSystemPath(destination, id); + LPCSTR runtimeTempDirectoryName = SharedMemoryManager::GetRuntimeTempDirectoryName(id); + VerifyStringOperation(AppendPathComponent(destination, runtimeTempDirectoryName)); +} - throw SharedMemoryException(static_cast(sharedMemoryError)); - } +void SharedMemoryHelpers::BuildSharedMemoryDirectoryPath(PathCharString& destination, const SharedMemoryId *id) +{ + BuildRuntimeRootPath(destination, id); + VerifyStringOperation(AppendPathComponent(destination, SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME)); } -void SharedMemoryHelpers::ReleaseFileLock(int fileDescriptor) +void SharedMemoryHelpers::BuildLockFilesDirectoryPath(PathCharString& destination, const SharedMemoryId *id) { - _ASSERTE(fileDescriptor != -1); + BuildRuntimeRootPath(destination, id); + VerifyStringOperation(AppendPathComponent(destination, SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME)); +} - int flockResult; - do +void SharedMemoryHelpers::BuildSessionDirectoryPath( + PathCharString& destination, + const SharedMemoryId *id, + bool isLockFile) +{ + if (isLockFile) { - flockResult = flock(fileDescriptor, LOCK_UN); - } while (flockResult != 0 && errno == EINTR); + BuildLockFilesDirectoryPath(destination, id); + } + else + { + BuildSharedMemoryDirectoryPath(destination, id); + } + + PathCharString sessionDirectoryName; + VerifyStringOperation(id->AppendSessionDirectoryName(sessionDirectoryName)); + VerifyStringOperation(AppendPathComponent(destination, sessionDirectoryName)); } -void SharedMemoryHelpers::BuildSharedFilesPath(PathCharString& destination, const char *suffix, int suffixCharCount) +void SharedMemoryHelpers::BuildPath(PathCharString& destination, const SharedMemoryId *id, bool isLockFile) { - _ASSERTE((int)strlen(suffix) == suffixCharCount); - - VerifyStringOperation(destination.Set(*gSharedFilesPath)); - VerifyStringOperation(destination.Append(suffix, suffixCharCount)); + BuildSessionDirectoryPath(destination, id, isLockFile); + VerifyStringOperation(AppendPathComponent(destination, id->GetName())); } bool SharedMemoryHelpers::AppendUInt32String( @@ -695,20 +1240,12 @@ void SharedMemoryHelpers::VerifyStringOperation(bool success) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // SharedMemoryId -SharedMemoryId::SharedMemoryId() : m_name(nullptr) +SharedMemoryId::SharedMemoryId() + : m_name(nullptr), m_nameCharCount(0), m_isSessionScope(false), m_isUserScope(false), m_userScopeUid((uid_t)0) { } -SharedMemoryId::SharedMemoryId(LPCSTR name, SIZE_T nameCharCount, bool isSessionScope) - : m_name(name), m_nameCharCount(nameCharCount), m_isSessionScope(isSessionScope) -{ - _ASSERTE(name != nullptr); - _ASSERTE(nameCharCount != 0); - _ASSERTE(nameCharCount <= SHARED_MEMORY_MAX_FILE_NAME_CHAR_COUNT); - _ASSERTE(strlen(name) == nameCharCount); -} - -SharedMemoryId::SharedMemoryId(LPCSTR name) +SharedMemoryId::SharedMemoryId(LPCSTR name, bool isUserScope) { _ASSERTE(name != nullptr); @@ -747,6 +1284,14 @@ SharedMemoryId::SharedMemoryId(LPCSTR name) throw SharedMemoryException(static_cast(SharedMemoryError::NameInvalid)); } } + + m_isUserScope = isUserScope; + m_userScopeUid = isUserScope ? geteuid() : (uid_t)0; + static_assert_no_msg(sizeof(uid_t) <= sizeof(UINT32)); + if ((uid_t)(UINT32)m_userScopeUid != m_userScopeUid) + { + throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + } } LPCSTR SharedMemoryId::GetName() const @@ -761,17 +1306,40 @@ SIZE_T SharedMemoryId::GetNameCharCount() const return m_nameCharCount; } +void SharedMemoryId::ReplaceNamePtr(LPCSTR name) +{ + _ASSERTE(name != nullptr); + _ASSERTE(m_nameCharCount != 0); + _ASSERTE(strlen(name) == m_nameCharCount); + m_name = name; +} + bool SharedMemoryId::IsSessionScope() const { _ASSERTE(m_name != nullptr); return m_isSessionScope; } -bool SharedMemoryId::Equals(SharedMemoryId *other) const +bool SharedMemoryId::IsUserScope() const +{ + _ASSERTE(m_name != nullptr); + return m_isUserScope; +} + +uid_t SharedMemoryId::GetUserScopeUid() const +{ + _ASSERTE(m_name != nullptr); + _ASSERTE(m_isUserScope); + return m_userScopeUid; +} + +bool SharedMemoryId::Equals(const SharedMemoryId *other) const { return GetNameCharCount() == other->GetNameCharCount() && IsSessionScope() == other->IsSessionScope() && + IsUserScope() == other->IsUserScope() && + (!IsUserScope() || GetUserScopeUid() == other->GetUserScopeUid()) && strcmp(GetName(), other->GetName()) == 0; } @@ -827,6 +1395,7 @@ void *SharedMemorySharedDataHeader::GetData() SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( SharedMemorySystemCallErrors *errors, LPCSTR name, + bool isUserScope, SharedMemorySharedDataHeader requiredSharedDataHeader, SIZE_T sharedDataByteCount, bool createIfNotExist, @@ -844,13 +1413,15 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( } PathCharString filePath; - SharedMemoryId id(name); + SharedMemoryId id(name, isUserScope); struct AutoCleanup { - bool m_acquiredCreationDeletionFileLock; - PathCharString *m_filePath; - SIZE_T m_sessionDirectoryPathCharCount; + const SharedMemoryId *m_acquiredCreationDeletionFileLockForId; + int m_sharedMemoryDirectoryFileDescriptor; + PathCharString *m_sessionDirectoryName; + int m_sessionDirectoryFileDescriptor; + LPCSTR m_fileName; bool m_createdFile; int m_fileDescriptor; bool m_acquiredFileLock; @@ -859,9 +1430,11 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( bool m_cancel; AutoCleanup() - : m_acquiredCreationDeletionFileLock(false), - m_filePath(nullptr), - m_sessionDirectoryPathCharCount(0), + : m_acquiredCreationDeletionFileLockForId(nullptr), + m_sharedMemoryDirectoryFileDescriptor(-1), + m_sessionDirectoryName(nullptr), + m_sessionDirectoryFileDescriptor(-1), + m_fileName(nullptr), m_createdFile(false), m_fileDescriptor(-1), m_acquiredFileLock(false), @@ -897,20 +1470,28 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( if (m_createdFile) { - _ASSERTE(m_filePath != nullptr); - unlink(*m_filePath); + _ASSERTE(m_sessionDirectoryFileDescriptor != -1); + _ASSERTE(m_fileName != nullptr); + unlinkat(m_sessionDirectoryFileDescriptor, m_fileName, 0); + } + + if (m_sessionDirectoryFileDescriptor != -1) + { + SharedMemoryHelpers::CloseFile(m_sessionDirectoryFileDescriptor); } - if (m_sessionDirectoryPathCharCount != 0) + if (m_sessionDirectoryName != nullptr) { - _ASSERTE(*m_filePath != nullptr); - m_filePath->CloseBuffer(m_sessionDirectoryPathCharCount); - rmdir(*m_filePath); + _ASSERTE(m_sharedMemoryDirectoryFileDescriptor != -1); + unlinkat( + m_sharedMemoryDirectoryFileDescriptor, + *m_sessionDirectoryName, + AT_REMOVEDIR); } - if (m_acquiredCreationDeletionFileLock) + if (m_acquiredCreationDeletionFileLockForId != nullptr) { - SharedMemoryManager::ReleaseCreationDeletionFileLock(); + SharedMemoryManager::ReleaseCreationDeletionFileLock(m_acquiredCreationDeletionFileLockForId); } } } autoCleanup; @@ -925,27 +1506,45 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( return processDataHeader; } - SharedMemoryManager::AcquireCreationDeletionFileLock(errors); - autoCleanup.m_acquiredCreationDeletionFileLock = true; + SharedMemoryManager::AcquireCreationDeletionFileLock(errors, &id); + autoCleanup.m_acquiredCreationDeletionFileLockForId = &id; + int sharedMemoryDirectoryFileDescriptor = + SharedMemoryManager::GetSharedMemoryDirectoryFileDescriptor(&id); // Create the session directory - SharedMemoryHelpers::VerifyStringOperation(SharedMemoryManager::CopySharedMemoryBasePath(filePath)); - SharedMemoryHelpers::VerifyStringOperation(filePath.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(id.AppendSessionDirectoryName(filePath)); - if (!SharedMemoryHelpers::EnsureDirectoryExists(errors, filePath, true /* isGlobalLockAcquired */, createIfNotExist)) + PathCharString sessionDirectoryName; + SharedMemoryHelpers::VerifyStringOperation(id.AppendSessionDirectoryName(sessionDirectoryName)); + SharedMemoryHelpers::BuildSessionDirectoryPath(filePath, &id, false /* isLockFile */); + int sessionDirectoryFileDescriptor = SharedMemoryHelpers::EnsureDirectoryExists( + errors, + sharedMemoryDirectoryFileDescriptor, + sessionDirectoryName, + filePath, + &id, + createIfNotExist); + if (sessionDirectoryFileDescriptor == -1) { _ASSERTE(!createIfNotExist); return nullptr; } - autoCleanup.m_filePath = &filePath; - autoCleanup.m_sessionDirectoryPathCharCount = filePath.GetCount(); + autoCleanup.m_sharedMemoryDirectoryFileDescriptor = sharedMemoryDirectoryFileDescriptor; + autoCleanup.m_sessionDirectoryName = &sessionDirectoryName; + autoCleanup.m_sessionDirectoryFileDescriptor = sessionDirectoryFileDescriptor; + autoCleanup.m_fileName = id.GetName(); // Create or open the shared memory file - SharedMemoryHelpers::VerifyStringOperation(filePath.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(filePath.Append(id.GetName(), id.GetNameCharCount())); + SharedMemoryHelpers::BuildPath(filePath, &id, false /* isLockFile */); bool createdFile; - int fileDescriptor = SharedMemoryHelpers::CreateOrOpenFile(errors, filePath, createIfNotExist, &createdFile); + int fileDescriptor = + SharedMemoryHelpers::CreateOrOpenFile( + errors, + sessionDirectoryFileDescriptor, + id.GetName(), + filePath, + &id, + createIfNotExist, + &createdFile); if (fileDescriptor == -1) { _ASSERTE(!createIfNotExist); @@ -1040,12 +1639,15 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen( // the shared data. The caller must release the file lock afterwards. if (!createdFile) { - autoCleanup.m_acquiredCreationDeletionFileLock = false; - SharedMemoryManager::ReleaseCreationDeletionFileLock(); + autoCleanup.m_acquiredCreationDeletionFileLockForId = nullptr; + SharedMemoryManager::ReleaseCreationDeletionFileLock(&id); } processDataHeader = SharedMemoryProcessDataHeader::New(&id, fileDescriptor, sharedDataHeader, sharedDataTotalByteCount); + SharedMemoryHelpers::CloseFile(sessionDirectoryFileDescriptor); + autoCleanup.m_sessionDirectoryFileDescriptor = -1; + autoCleanup.m_sessionDirectoryName = nullptr; autoCleanup.m_cancel = true; if (createdFile) { @@ -1110,7 +1712,7 @@ void SharedMemoryProcessDataHeader::PalObject_Close( } SharedMemoryProcessDataHeader::SharedMemoryProcessDataHeader( - SharedMemoryId *id, + const SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount) @@ -1133,13 +1735,14 @@ SharedMemoryProcessDataHeader::SharedMemoryProcessDataHeader( char *nameCopy = reinterpret_cast(this + 1); SIZE_T nameByteCount = id->GetNameCharCount() + 1; memcpy_s(nameCopy, nameByteCount, id->GetName(), nameByteCount); - m_id = SharedMemoryId(nameCopy, id->GetNameCharCount(), id->IsSessionScope()); + m_id = *id; + m_id.ReplaceNamePtr(nameCopy); SharedMemoryManager::AddProcessDataHeader(this); } SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::New( - SharedMemoryId *id, + const SharedMemoryId *id, int fileDescriptor, SharedMemorySharedDataHeader *sharedDataHeader, SIZE_T sharedDataTotalByteCount) @@ -1179,17 +1782,17 @@ void SharedMemoryProcessDataHeader::Close() struct AutoReleaseCreationDeletionFileLock { - bool m_acquired; + const SharedMemoryId *m_acquiredForId; - AutoReleaseCreationDeletionFileLock() : m_acquired(false) + AutoReleaseCreationDeletionFileLock() : m_acquiredForId(nullptr) { } ~AutoReleaseCreationDeletionFileLock() { - if (m_acquired) + if (m_acquiredForId != nullptr) { - SharedMemoryManager::ReleaseCreationDeletionFileLock(); + SharedMemoryManager::ReleaseCreationDeletionFileLock(m_acquiredForId); } } } autoReleaseCreationDeletionFileLock; @@ -1203,8 +1806,8 @@ void SharedMemoryProcessDataHeader::Close() bool releaseSharedData = false; try { - SharedMemoryManager::AcquireCreationDeletionFileLock(nullptr); - autoReleaseCreationDeletionFileLock.m_acquired = true; + SharedMemoryManager::AcquireCreationDeletionFileLock(nullptr, GetId()); + autoReleaseCreationDeletionFileLock.m_acquiredForId = GetId(); SharedMemoryHelpers::ReleaseFileLock(m_fileDescriptor); if (SharedMemoryHelpers::TryAcquireFileLock(nullptr, m_fileDescriptor, LOCK_EX | LOCK_NB)) @@ -1246,18 +1849,30 @@ void SharedMemoryProcessDataHeader::Close() try { - // Delete the shared memory file, and the session directory if it's not empty - PathCharString path; - SharedMemoryHelpers::VerifyStringOperation(SharedMemoryManager::CopySharedMemoryBasePath(path)); - SharedMemoryHelpers::VerifyStringOperation(path.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(m_id.AppendSessionDirectoryName(path)); - SharedMemoryHelpers::VerifyStringOperation(path.Append('/')); - - SIZE_T sessionDirectoryPathCharCount = path.GetCount(); - SharedMemoryHelpers::VerifyStringOperation(path.Append(m_id.GetName(), m_id.GetNameCharCount())); - unlink(path); - path.CloseBuffer(sessionDirectoryPathCharCount); - rmdir(path); + // Delete the shared memory file, and the session directory if it's empty. + int sharedMemoryDirectoryFileDescriptor = + SharedMemoryManager::GetSharedMemoryDirectoryFileDescriptor(&m_id); + PathCharString sessionDirectoryName; + SharedMemoryHelpers::VerifyStringOperation(m_id.AppendSessionDirectoryName(sessionDirectoryName)); + PathCharString sessionDirectoryPath; + SharedMemoryHelpers::BuildSessionDirectoryPath( + sessionDirectoryPath, + &m_id, + false /* isLockFile */); + int sessionDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + nullptr, + sharedMemoryDirectoryFileDescriptor, + sessionDirectoryName, + sessionDirectoryPath, + &m_id, + false /* createIfNotExist */); + if (sessionDirectoryFileDescriptor != -1) + { + unlinkat(sessionDirectoryFileDescriptor, m_id.GetName(), 0); + SharedMemoryHelpers::CloseFile(sessionDirectoryFileDescriptor); + } + unlinkat(sharedMemoryDirectoryFileDescriptor, sessionDirectoryName, AT_REMOVEDIR); } catch (SharedMemoryException) { @@ -1265,7 +1880,7 @@ void SharedMemoryProcessDataHeader::Close() } } -SharedMemoryId *SharedMemoryProcessDataHeader::GetId() +const SharedMemoryId *SharedMemoryProcessDataHeader::GetId() const { return &m_id; } @@ -1345,39 +1960,23 @@ void SharedMemoryProcessDataHeader::DecRefCount() // SharedMemoryManager CRITICAL_SECTION SharedMemoryManager::s_creationDeletionProcessLock; +int SharedMemoryManager::s_runtimeDirectoryFileDescriptor = -1; int SharedMemoryManager::s_creationDeletionLockFileDescriptor = -1; +SharedMemoryManager::UserScopeUidData *SharedMemoryManager::s_userScopeUidData = nullptr; +int SharedMemoryManager::s_userScopeUidDataCount = 0; +int SharedMemoryManager::s_userScopeUidDataCapacity = 0; + SharedMemoryProcessDataHeader *SharedMemoryManager::s_processDataHeaderListHead = nullptr; -PathCharString* SharedMemoryManager::s_runtimeTempDirectoryPath; -PathCharString* SharedMemoryManager::s_sharedMemoryDirectoryPath; #ifdef _DEBUG SIZE_T SharedMemoryManager::s_creationDeletionProcessLockOwnerThreadId = SharedMemoryHelpers::InvalidThreadId; SIZE_T SharedMemoryManager::s_creationDeletionFileLockOwnerThreadId = SharedMemoryHelpers::InvalidThreadId; #endif // _DEBUG -bool SharedMemoryManager::StaticInitialize() +void SharedMemoryManager::StaticInitialize() { InitializeCriticalSection(&s_creationDeletionProcessLock); - - s_runtimeTempDirectoryPath = InternalNew(); - s_sharedMemoryDirectoryPath = InternalNew(); - - if (s_runtimeTempDirectoryPath && s_sharedMemoryDirectoryPath) - { - try - { - SharedMemoryHelpers::BuildSharedFilesPath(*s_runtimeTempDirectoryPath, SHARED_MEMORY_RUNTIME_TEMP_DIRECTORY_NAME); - SharedMemoryHelpers::BuildSharedFilesPath(*s_sharedMemoryDirectoryPath, SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME); - - return true; - } - catch (SharedMemoryException) - { - // Ignore and let function fail - } - } - return false; } void SharedMemoryManager::StaticClose() @@ -1419,72 +2018,276 @@ void SharedMemoryManager::ReleaseCreationDeletionProcessLock() LeaveCriticalSection(&s_creationDeletionProcessLock); } -void SharedMemoryManager::AcquireCreationDeletionFileLock(SharedMemorySystemCallErrors *errors) +void SharedMemoryManager::AcquireCreationDeletionFileLock( + SharedMemorySystemCallErrors *errors, + const SharedMemoryId *id) { + _ASSERTE(id != nullptr); _ASSERTE(IsCreationDeletionProcessLockAcquired()); _ASSERTE(!IsCreationDeletionFileLockAcquired()); - if (s_creationDeletionLockFileDescriptor == -1) + int creationDeletionLockFileDescriptor = GetSharedMemoryDirectoryFileDescriptor(id); + if (creationDeletionLockFileDescriptor == -1) { - if (!SharedMemoryHelpers::EnsureDirectoryExists( - errors, - *gSharedFilesPath, - false /* isGlobalLockAcquired */, - false /* createIfNotExist */, - true /* isSystemDirectory */)) + PathCharString sharedFilesSystemPath; + SharedMemoryHelpers::BuildSharedFilesSystemPath(sharedFilesSystemPath, id); + if (id->IsUserScope()) { - _ASSERTE(errno == ENOENT); - if (errors != nullptr) + SIZE_T homeDirectoryPathCharCount = + GetUserScopePersistedHomeDirectoryPathCharCount(id->GetUserScopeUid()); + if (homeDirectoryPathCharCount != 0) { - errors->Append("stat(\"%s\", ...) == -1; errno == ENOENT;", (const char *)*gSharedFilesPath); + SharedMemoryHelpers::EnsurePersistedUserDirectoryExists( + errors, + sharedFilesSystemPath, + homeDirectoryPathCharCount, + id); } - - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); } - SharedMemoryHelpers::EnsureDirectoryExists( - errors, - *s_runtimeTempDirectoryPath, - false /* isGlobalLockAcquired */); - - SharedMemoryHelpers::EnsureDirectoryExists( - errors, - *s_sharedMemoryDirectoryPath, - false /* isGlobalLockAcquired */); - - s_creationDeletionLockFileDescriptor = SharedMemoryHelpers::OpenDirectory(errors, *s_sharedMemoryDirectoryPath); - if (s_creationDeletionLockFileDescriptor == -1) + int sharedFilesSystemDirectoryFileDescriptor = + SharedMemoryHelpers::OpenSharedFilesSystemDirectory(errors, id); + int runtimeDirectoryFileDescriptor = -1; + try { - if (errors != nullptr) + PathCharString runtimeDirectoryPath; + SharedMemoryHelpers::BuildRuntimeRootPath(runtimeDirectoryPath, id); + runtimeDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + errors, + sharedFilesSystemDirectoryFileDescriptor, + GetRuntimeTempDirectoryName(id), + runtimeDirectoryPath, + id); + + PathCharString sharedMemoryDirectoryPath; + SharedMemoryHelpers::BuildSharedMemoryDirectoryPath(sharedMemoryDirectoryPath, id); + creationDeletionLockFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + errors, + runtimeDirectoryFileDescriptor, + SHARED_MEMORY_SHARED_MEMORY_DIRECTORY_NAME, + sharedMemoryDirectoryPath, + id); + } + catch (SharedMemoryException) + { + SharedMemoryHelpers::CloseFile(sharedFilesSystemDirectoryFileDescriptor); + if (runtimeDirectoryFileDescriptor != -1) { - int errorCode = errno; - errors->Append( - "open(\"%s\", O_RDONLY | O_CLOEXEC, 0) == -1; errno == %s;", - (const char *)*s_sharedMemoryDirectoryPath, - GetFriendlyErrorCodeString(errorCode)); + SharedMemoryHelpers::CloseFile(runtimeDirectoryFileDescriptor); } + throw; + } + SharedMemoryHelpers::CloseFile(sharedFilesSystemDirectoryFileDescriptor); - throw SharedMemoryException(static_cast(SharedMemoryError::IO)); + if (id->IsUserScope()) + { + try + { + SetUserScopeUidDirectoryFileDescriptors( + id->GetUserScopeUid(), + runtimeDirectoryFileDescriptor, + creationDeletionLockFileDescriptor); + } + catch (SharedMemoryException) + { + SharedMemoryHelpers::CloseFile(runtimeDirectoryFileDescriptor); + SharedMemoryHelpers::CloseFile(creationDeletionLockFileDescriptor); + throw; + } + } + else + { + s_runtimeDirectoryFileDescriptor = runtimeDirectoryFileDescriptor; + s_creationDeletionLockFileDescriptor = creationDeletionLockFileDescriptor; } } - bool acquiredFileLock = SharedMemoryHelpers::TryAcquireFileLock(errors, s_creationDeletionLockFileDescriptor, LOCK_EX); + bool acquiredFileLock = + SharedMemoryHelpers::TryAcquireFileLock(errors, creationDeletionLockFileDescriptor, LOCK_EX); _ASSERTE(acquiredFileLock); #ifdef _DEBUG s_creationDeletionFileLockOwnerThreadId = THREADSilentGetCurrentThreadId(); #endif // _DEBUG } -void SharedMemoryManager::ReleaseCreationDeletionFileLock() +void SharedMemoryManager::ReleaseCreationDeletionFileLock(const SharedMemoryId *id) { + _ASSERTE(id != nullptr); _ASSERTE(IsCreationDeletionProcessLockAcquired()); _ASSERTE(IsCreationDeletionFileLockAcquired()); - _ASSERTE(s_creationDeletionLockFileDescriptor != -1); + + int creationDeletionLockFileDescriptor = GetSharedMemoryDirectoryFileDescriptor(id); + _ASSERTE(creationDeletionLockFileDescriptor != -1); #ifdef _DEBUG s_creationDeletionFileLockOwnerThreadId = SharedMemoryHelpers::InvalidThreadId; #endif // _DEBUG - SharedMemoryHelpers::ReleaseFileLock(s_creationDeletionLockFileDescriptor); + SharedMemoryHelpers::ReleaseFileLock(creationDeletionLockFileDescriptor); +} + +SharedMemoryManager::UserScopeUidData *SharedMemoryManager::GetOrCreateUserScopeUidData(uid_t userScopeUid) +{ + _ASSERTE(IsCreationDeletionProcessLockAcquired()); + + // Retain the selected base together with this UID's creation/deletion lock so environment changes cannot split them. + UserScopeUidData *entry = FindUserScopeUidData(userScopeUid); + if (entry != nullptr) + { + return entry; + } + + PathCharString sharedFilesPath; + SIZE_T persistedUserHomeDirectoryPathCharCount; + bool isApplicationContainer; + SharedMemoryHelpers::BuildUserSpecificSharedFilesPath( + sharedFilesPath, + userScopeUid, + &persistedUserHomeDirectoryPathCharCount, + &isApplicationContainer); + + SIZE_T pathCharCount = sharedFilesPath.GetCount(); + _ASSERTE(pathCharCount != 0); + const char *path = sharedFilesPath; + while (pathCharCount > 1 && path[pathCharCount - 1] == '/') + { + sharedFilesPath.CloseBuffer(--pathCharCount); + path = sharedFilesPath; + } + if (pathCharCount == static_cast(-1)) + { + throw SharedMemoryException(static_cast(SharedMemoryError::OutOfMemory)); + } + + char *sharedFilesPathCopy = reinterpret_cast(SharedMemoryHelpers::Alloc(pathCharCount + 1)); + AutoFreeBuffer autoFreeSharedFilesPathCopy(sharedFilesPathCopy); + memcpy_s(sharedFilesPathCopy, pathCharCount + 1, path, pathCharCount + 1); + + int count = s_userScopeUidDataCount; + int capacity = s_userScopeUidDataCapacity; + if (count >= capacity) + { + int newCapacity = capacity == 0 ? 1 : capacity * 2; + if (newCapacity <= capacity || + static_cast(newCapacity) > + static_cast(-1) / sizeof(UserScopeUidData)) + { + throw SharedMemoryException(static_cast(SharedMemoryError::OutOfMemory)); + } + + UserScopeUidData *newArray = InternalNewArray(newCapacity); + if (newArray == nullptr) + { + throw SharedMemoryException(static_cast(SharedMemoryError::OutOfMemory)); + } + + if (count != 0) + { + CopyMemory( + newArray, + s_userScopeUidData, + count * sizeof(UserScopeUidData)); + InternalDeleteArray(s_userScopeUidData); + } + + s_userScopeUidData = newArray; + s_userScopeUidDataCapacity = newCapacity; + } + + entry = &s_userScopeUidData[count]; + *entry = + UserScopeUidData( + userScopeUid, + sharedFilesPathCopy, + persistedUserHomeDirectoryPathCharCount, + isApplicationContainer); + s_userScopeUidDataCount = count + 1; + autoFreeSharedFilesPathCopy.Cancel(); + return entry; +} + +SharedMemoryManager::UserScopeUidData *SharedMemoryManager::FindUserScopeUidData(uid_t userScopeUid) +{ + _ASSERTE(IsCreationDeletionProcessLockAcquired()); + + for (int i = 0; i < s_userScopeUidDataCount; i++) + { + UserScopeUidData *entry = &s_userScopeUidData[i]; + _ASSERTE(entry->sharedFilesPath != nullptr); + if (entry->userScopeUid == userScopeUid) + { + return entry; + } + } + + return nullptr; +} + +LPCSTR SharedMemoryManager::GetUserScopeSharedFilesPath(uid_t userScopeUid) +{ + return GetOrCreateUserScopeUidData(userScopeUid)->sharedFilesPath; +} + +SIZE_T SharedMemoryManager::GetUserScopePersistedHomeDirectoryPathCharCount(uid_t userScopeUid) +{ + return GetOrCreateUserScopeUidData(userScopeUid)->persistedUserHomeDirectoryPathCharCount; +} + +LPCSTR SharedMemoryManager::GetRuntimeTempDirectoryName(const SharedMemoryId *id) +{ + _ASSERTE(id != nullptr); + + // An Apple application-group container is already user-specific, so user-unscoped and user-scoped mutexes would + // otherwise use the same .dotnet directory and conflict. Give user-scoped mutexes a parallel directory to preserve + // the distinction between the scopes. + return id->IsUserScope() && GetOrCreateUserScopeUidData(id->GetUserScopeUid())->isApplicationContainer + ? SHARED_MEMORY_USER_RUNTIME_TEMP_DIRECTORY_NAME + : SHARED_MEMORY_RUNTIME_TEMP_DIRECTORY_NAME; +} + +void SharedMemoryManager::SetUserScopeUidDirectoryFileDescriptors( + uid_t userScopeUid, + int runtimeDirectoryFileDescriptor, + int sharedMemoryDirectoryFileDescriptor) +{ + _ASSERTE(IsCreationDeletionProcessLockAcquired()); + _ASSERTE(runtimeDirectoryFileDescriptor != -1); + _ASSERTE(sharedMemoryDirectoryFileDescriptor != -1); + + UserScopeUidData *entry = GetOrCreateUserScopeUidData(userScopeUid); + _ASSERTE(entry->runtimeDirectoryFileDescriptor == -1); + _ASSERTE(entry->sharedMemoryDirectoryFileDescriptor == -1); + entry->runtimeDirectoryFileDescriptor = runtimeDirectoryFileDescriptor; + entry->sharedMemoryDirectoryFileDescriptor = sharedMemoryDirectoryFileDescriptor; +} + +int SharedMemoryManager::GetRuntimeDirectoryFileDescriptor(const SharedMemoryId *id) +{ + _ASSERTE(id != nullptr); + _ASSERTE(IsCreationDeletionProcessLockAcquired()); + + if (!id->IsUserScope()) + { + return s_runtimeDirectoryFileDescriptor; + } + + UserScopeUidData *entry = FindUserScopeUidData(id->GetUserScopeUid()); + return entry == nullptr ? -1 : entry->runtimeDirectoryFileDescriptor; +} + +int SharedMemoryManager::GetSharedMemoryDirectoryFileDescriptor(const SharedMemoryId *id) +{ + _ASSERTE(id != nullptr); + _ASSERTE(IsCreationDeletionProcessLockAcquired()); + + if (!id->IsUserScope()) + { + return s_creationDeletionLockFileDescriptor; + } + + UserScopeUidData *entry = FindUserScopeUidData(id->GetUserScopeUid()); + return entry == nullptr ? -1 : entry->sharedMemoryDirectoryFileDescriptor; } #ifdef _DEBUG @@ -1537,7 +2340,7 @@ void SharedMemoryManager::RemoveProcessDataHeader(SharedMemoryProcessDataHeader _ASSERTE(false); } -SharedMemoryProcessDataHeader *SharedMemoryManager::FindProcessDataHeader(SharedMemoryId *id) +SharedMemoryProcessDataHeader *SharedMemoryManager::FindProcessDataHeader(const SharedMemoryId *id) { _ASSERTE(IsCreationDeletionProcessLockAcquired()); @@ -1553,8 +2356,3 @@ SharedMemoryProcessDataHeader *SharedMemoryManager::FindProcessDataHeader(Shared } return nullptr; } - -bool SharedMemoryManager::CopySharedMemoryBasePath(PathCharString& destination) -{ - return destination.Set(*s_sharedMemoryDirectoryPath) != FALSE; -} diff --git a/src/coreclr/pal/src/synchobj/mutex.cpp b/src/coreclr/pal/src/synchobj/mutex.cpp index e3d63fe232b49f..38921fe8dab904 100644 --- a/src/coreclr/pal/src/synchobj/mutex.cpp +++ b/src/coreclr/pal/src/synchobj/mutex.cpp @@ -34,8 +34,6 @@ SET_DEFAULT_DEBUG_CHANNEL(SYNC); // some headers have code with asserts, so do t #include #include -#include "pal/sharedmemory.inl" - using namespace CorUnix; /* ------------------- Definitions ------------------------------*/ @@ -103,7 +101,7 @@ CreateMutexW( IN BOOL bInitialOwner, IN LPCWSTR lpName) { - return PAL_CreateMutexW(bInitialOwner, lpName, nullptr, 0); + return PAL_CreateMutexW(bInitialOwner, lpName, false /* bCurrentUserOnly */, nullptr, 0); } /*++ @@ -127,6 +125,7 @@ PALAPI PAL_CreateMutexW( IN BOOL bInitialOwner, IN LPCWSTR lpName, + IN BOOL bCurrentUserOnly, IN LPSTR lpSystemCallErrors, IN DWORD dwSystemCallErrorsBufferSize) { @@ -136,10 +135,11 @@ PAL_CreateMutexW( char utf8Name[SHARED_MEMORY_MAX_NAME_CHAR_COUNT + 1]; PERF_ENTRY(PAL_CreateMutexW); - ENTRY("PAL_CreateMutexW(bInitialOwner=%d, lpName=%p (%S), lpSystemCallErrors=%p, dwSystemCallErrorsBufferSize=%d\n", + ENTRY("PAL_CreateMutexW(bInitialOwner=%d, lpName=%p (%S), bCurrentUserOnly=%d, lpSystemCallErrors=%p, dwSystemCallErrorsBufferSize=%d\n", bInitialOwner, lpName, lpName?lpName:W16_NULLSTRING, + bCurrentUserOnly, lpSystemCallErrors, dwSystemCallErrorsBufferSize); @@ -185,6 +185,7 @@ PAL_CreateMutexW( nullptr, bInitialOwner, lpName == nullptr ? nullptr : utf8Name, + bCurrentUserOnly, &hMutex ); } @@ -252,6 +253,7 @@ CorUnix::InternalCreateMutex( LPSECURITY_ATTRIBUTES lpMutexAttributes, BOOL bInitialOwner, LPCSTR lpName, + BOOL bCurrentUserOnly, HANDLE *phMutex ) { @@ -267,11 +269,12 @@ CorUnix::InternalCreateMutex( _ASSERTE(NULL != phMutex); ENTRY("InternalCreateMutex(pthr=%p, lpMutexAttributes=%p, bInitialOwner=%d" - ", lpName=%p, phMutex=%p)\n", + ", lpName=%p, bCurrentUserOnly=%d, phMutex=%p)\n", pthr, lpMutexAttributes, bInitialOwner, lpName, + bCurrentUserOnly, phMutex ); @@ -331,7 +334,13 @@ CorUnix::InternalCreateMutex( SharedMemoryProcessDataHeader *processDataHeader; try { - processDataHeader = NamedMutexProcessData::CreateOrOpen(errors, lpName, !!bInitialOwner, &createdNamedMutex); + processDataHeader = + NamedMutexProcessData::CreateOrOpen( + errors, + lpName, + !!bCurrentUserOnly, + !!bInitialOwner, + &createdNamedMutex); } catch (SharedMemoryException ex) { @@ -557,7 +566,7 @@ OpenMutexA ( goto OpenMutexAExit; } - palError = InternalOpenMutex(nullptr, pthr, lpName, &hMutex); + palError = InternalOpenMutex(nullptr, pthr, lpName, false /* bCurrentUserOnly */, &hMutex); OpenMutexAExit: if (NO_ERROR != palError) @@ -585,7 +594,7 @@ OpenMutexW( IN BOOL bInheritHandle, IN LPCWSTR lpName) { - return PAL_OpenMutexW(lpName, nullptr, 0); + return PAL_OpenMutexW(lpName, false /* bCurrentUserOnly */, nullptr, 0); } /*++ @@ -607,6 +616,7 @@ HANDLE PALAPI PAL_OpenMutexW( IN LPCWSTR lpName, + IN BOOL bCurrentUserOnly, IN LPSTR lpSystemCallErrors, IN DWORD dwSystemCallErrorsBufferSize) { @@ -616,9 +626,10 @@ PAL_OpenMutexW( char utf8Name[SHARED_MEMORY_MAX_NAME_CHAR_COUNT + 1]; PERF_ENTRY(PAL_OpenMutexW); - ENTRY("PAL_OpenMutexW(lpName=%p (%S), lpSystemCallErrors=%p, dwSystemCallErrorsBufferSize=%d)\n", + ENTRY("PAL_OpenMutexW(lpName=%p (%S), bCurrentUserOnly=%d, lpSystemCallErrors=%p, dwSystemCallErrorsBufferSize=%d)\n", lpName, lpName?lpName:W16_NULLSTRING, + bCurrentUserOnly, lpSystemCallErrors, dwSystemCallErrorsBufferSize); @@ -657,7 +668,13 @@ PAL_OpenMutexW( } SharedMemorySystemCallErrors errors(lpSystemCallErrors, (int)dwSystemCallErrorsBufferSize); - palError = InternalOpenMutex(&errors, pthr, lpName == nullptr ? nullptr : utf8Name, &hMutex); + palError = + InternalOpenMutex( + &errors, + pthr, + lpName == nullptr ? nullptr : utf8Name, + bCurrentUserOnly, + &hMutex); } OpenMutexWExit: @@ -689,6 +706,7 @@ CorUnix::InternalOpenMutex( SharedMemorySystemCallErrors *errors, CPalThread *pthr, LPCSTR lpName, + BOOL bCurrentUserOnly, HANDLE *phMutex ) { @@ -703,9 +721,10 @@ CorUnix::InternalOpenMutex( _ASSERTE(NULL != phMutex); ENTRY("InternalOpenMutex(pthr=%p, " - "lpName=%p, phMutex=%p)\n", + "lpName=%p, bCurrentUserOnly=%d, phMutex=%p)\n", pthr, lpName, + bCurrentUserOnly, phMutex ); @@ -725,7 +744,7 @@ CorUnix::InternalOpenMutex( SharedMemoryProcessDataHeader *processDataHeader; try { - processDataHeader = NamedMutexProcessData::Open(errors, lpName); + processDataHeader = NamedMutexProcessData::Open(errors, lpName, !!bCurrentUserOnly); } catch (SharedMemoryException ex) { @@ -1088,20 +1107,39 @@ const DWORD NamedMutexProcessData::PollLoopMaximumSleepMilliseconds = 100; SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( SharedMemorySystemCallErrors *errors, LPCSTR name, + bool isUserScope, bool acquireLockIfCreated, bool *createdRef) { - return CreateOrOpen(errors, name, true /* createIfNotExist */, acquireLockIfCreated, createdRef); + return + CreateOrOpen( + errors, + name, + isUserScope, + true /* createIfNotExist */, + acquireLockIfCreated, + createdRef); } -SharedMemoryProcessDataHeader *NamedMutexProcessData::Open(SharedMemorySystemCallErrors *errors, LPCSTR name) +SharedMemoryProcessDataHeader *NamedMutexProcessData::Open( + SharedMemorySystemCallErrors *errors, + LPCSTR name, + bool isUserScope) { - return CreateOrOpen(errors, name, false /* createIfNotExist */, false /* acquireLockIfCreated */, nullptr /* createdRef */); + return + CreateOrOpen( + errors, + name, + isUserScope, + false /* createIfNotExist */, + false /* acquireLockIfCreated */, + nullptr /* createdRef */); } SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( SharedMemorySystemCallErrors *errors, LPCSTR name, + bool isUserScope, bool createIfNotExist, bool acquireLockIfCreated, bool *createdRef) @@ -1119,8 +1157,10 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( bool m_acquiredCreationDeletionFileLock; SharedMemoryProcessDataHeader *m_processDataHeader; #if !NAMED_MUTEX_USE_PTHREAD_MUTEX - PathCharString *m_lockFilePath; - SIZE_T m_sessionDirectoryPathCharCount; + int m_lockFilesDirectoryFileDescriptor; + PathCharString *m_sessionDirectoryName; + int m_sessionDirectoryFileDescriptor; + LPCSTR m_fileName; bool m_createdLockFile; int m_lockFileDescriptor; #endif // !NAMED_MUTEX_USE_PTHREAD_MUTEX @@ -1131,8 +1171,10 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( m_acquiredCreationDeletionFileLock(false), m_processDataHeader(nullptr), #if !NAMED_MUTEX_USE_PTHREAD_MUTEX - m_lockFilePath(nullptr), - m_sessionDirectoryPathCharCount(0), + m_lockFilesDirectoryFileDescriptor(-1), + m_sessionDirectoryName(nullptr), + m_sessionDirectoryFileDescriptor(-1), + m_fileName(nullptr), m_createdLockFile(false), m_lockFileDescriptor(-1), #endif // !NAMED_MUTEX_USE_PTHREAD_MUTEX @@ -1152,22 +1194,36 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( if (m_createdLockFile) { - _ASSERTE(m_lockFilePath != nullptr); - unlink(*m_lockFilePath); + _ASSERTE(m_sessionDirectoryFileDescriptor != -1); + _ASSERTE(m_fileName != nullptr); + unlinkat(m_sessionDirectoryFileDescriptor, m_fileName, 0); + } + + if (m_sessionDirectoryFileDescriptor != -1) + { + SharedMemoryHelpers::CloseFile(m_sessionDirectoryFileDescriptor); + } + + if (m_sessionDirectoryName != nullptr) + { + _ASSERTE(m_lockFilesDirectoryFileDescriptor != -1); + unlinkat( + m_lockFilesDirectoryFileDescriptor, + *m_sessionDirectoryName, + AT_REMOVEDIR); } - if (m_sessionDirectoryPathCharCount != 0) + if (m_lockFilesDirectoryFileDescriptor != -1) { - _ASSERTE(m_lockFilePath != nullptr); - m_lockFilePath->CloseBuffer(m_sessionDirectoryPathCharCount); - rmdir(*m_lockFilePath); + SharedMemoryHelpers::CloseFile(m_lockFilesDirectoryFileDescriptor); } } #endif // !NAMED_MUTEX_USE_PTHREAD_MUTEX if (m_acquiredCreationDeletionFileLock) { - SharedMemoryManager::ReleaseCreationDeletionFileLock(); + _ASSERTE(m_processDataHeader != nullptr); + SharedMemoryManager::ReleaseCreationDeletionFileLock(m_processDataHeader->GetId()); } if (!m_cancel && m_processDataHeader != nullptr) @@ -1192,6 +1248,7 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( SharedMemoryProcessDataHeader::CreateOrOpen( errors, name, + isUserScope, SharedMemorySharedDataHeader(SharedMemoryType::Mutex, SyncSystemVersion), sizeof(NamedMutexSharedData), createIfNotExist, @@ -1200,6 +1257,13 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( { *createdRef = created; } + if (processDataHeader == nullptr) + { + _ASSERTE(!created); + _ASSERTE(!createIfNotExist); + return nullptr; + } + autoCleanup.m_processDataHeader = processDataHeader; if (created) { // If the shared memory file was created, the creation/deletion file lock would have been acquired so that we can @@ -1207,12 +1271,6 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( _ASSERTE(SharedMemoryManager::IsCreationDeletionFileLockAcquired()); autoCleanup.m_acquiredCreationDeletionFileLock = true; } - if (processDataHeader == nullptr) - { - _ASSERTE(!createIfNotExist); - return nullptr; - } - autoCleanup.m_processDataHeader = processDataHeader; if (created) { @@ -1223,28 +1281,56 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( if (processDataHeader->GetData() == nullptr) { #if !NAMED_MUTEX_USE_PTHREAD_MUTEX + const SharedMemoryId *id = processDataHeader->GetId(); + // Create the lock files directory - SharedMemoryHelpers::BuildSharedFilesPath(lockFilePath, SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME); - if (created) + SharedMemoryHelpers::BuildLockFilesDirectoryPath(lockFilePath, id); + int lockFilesDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + errors, + SharedMemoryManager::GetRuntimeDirectoryFileDescriptor(id), + SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME, + lockFilePath, + id, + created); + if (lockFilesDirectoryFileDescriptor == -1) { - SharedMemoryHelpers::EnsureDirectoryExists(errors, lockFilePath, true /* isGlobalLockAcquired */); + _ASSERTE(!created); + return nullptr; } + autoCleanup.m_lockFilesDirectoryFileDescriptor = lockFilesDirectoryFileDescriptor; // Create the session directory - SharedMemoryId *id = processDataHeader->GetId(); - SharedMemoryHelpers::VerifyStringOperation(lockFilePath.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(id->AppendSessionDirectoryName(lockFilePath)); - if (created) + PathCharString sessionDirectoryName; + SharedMemoryHelpers::VerifyStringOperation(id->AppendSessionDirectoryName(sessionDirectoryName)); + SharedMemoryHelpers::BuildSessionDirectoryPath(lockFilePath, id, true /* isLockFile */); + int sessionDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + errors, + lockFilesDirectoryFileDescriptor, + sessionDirectoryName, + lockFilePath, + id, + created); + if (sessionDirectoryFileDescriptor == -1) { - SharedMemoryHelpers::EnsureDirectoryExists(errors, lockFilePath, true /* isGlobalLockAcquired */); - autoCleanup.m_lockFilePath = &lockFilePath; - autoCleanup.m_sessionDirectoryPathCharCount = lockFilePath.GetCount(); + _ASSERTE(!created); + return nullptr; } + autoCleanup.m_sessionDirectoryName = &sessionDirectoryName; + autoCleanup.m_sessionDirectoryFileDescriptor = sessionDirectoryFileDescriptor; + autoCleanup.m_fileName = id->GetName(); // Create or open the lock file - SharedMemoryHelpers::VerifyStringOperation(lockFilePath.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(lockFilePath.Append(id->GetName(), id->GetNameCharCount())); - int lockFileDescriptor = SharedMemoryHelpers::CreateOrOpenFile(errors, lockFilePath, created); + SharedMemoryHelpers::BuildPath(lockFilePath, id, true /* isLockFile */); + int lockFileDescriptor = + SharedMemoryHelpers::CreateOrOpenFile( + errors, + sessionDirectoryFileDescriptor, + id->GetName(), + lockFilePath, + id, + created); if (lockFileDescriptor == -1) { _ASSERTE(!created); @@ -1287,6 +1373,14 @@ SharedMemoryProcessDataHeader *NamedMutexProcessData::CreateOrOpen( MutexTryAcquireLockResult tryAcquireLockResult = processData->TryAcquireLock(errors, 0); _ASSERTE(tryAcquireLockResult == MutexTryAcquireLockResult::AcquiredLock); } + + #if !NAMED_MUTEX_USE_PTHREAD_MUTEX + SharedMemoryHelpers::CloseFile(sessionDirectoryFileDescriptor); + autoCleanup.m_sessionDirectoryFileDescriptor = -1; + autoCleanup.m_sessionDirectoryName = nullptr; + SharedMemoryHelpers::CloseFile(lockFilesDirectoryFileDescriptor); + autoCleanup.m_lockFilesDirectoryFileDescriptor = -1; + #endif // !NAMED_MUTEX_USE_PTHREAD_MUTEX } autoCleanup.m_cancel = true; @@ -1404,25 +1498,62 @@ void NamedMutexProcessData::Close(bool isAbruptShutdown, bool releaseSharedData) return; } + int lockFilesDirectoryFileDescriptor = -1; + int sessionDirectoryFileDescriptor = -1; try { - // Delete the lock file, and the session directory if it's not empty - PathCharString path; - SharedMemoryHelpers::BuildSharedFilesPath(path, SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME); - SharedMemoryId *id = m_processDataHeader->GetId(); - SharedMemoryHelpers::VerifyStringOperation(path.Append('/')); - SharedMemoryHelpers::VerifyStringOperation(id->AppendSessionDirectoryName(path)); - SharedMemoryHelpers::VerifyStringOperation(path.Append('/')); - SIZE_T sessionDirectoryPathCharCount = path.GetCount(); - SharedMemoryHelpers::VerifyStringOperation(path.Append(id->GetName(), id->GetNameCharCount())); - unlink(path); - path.CloseBuffer(sessionDirectoryPathCharCount); - rmdir(path); + // Delete the lock file, and the session directory if it's empty. + const SharedMemoryId *id = m_processDataHeader->GetId(); + PathCharString lockFilesDirectoryPath; + SharedMemoryHelpers::BuildLockFilesDirectoryPath(lockFilesDirectoryPath, id); + lockFilesDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + nullptr, + SharedMemoryManager::GetRuntimeDirectoryFileDescriptor(id), + SHARED_MEMORY_LOCK_FILES_DIRECTORY_NAME, + lockFilesDirectoryPath, + id, + false /* createIfNotExist */); + if (lockFilesDirectoryFileDescriptor != -1) + { + PathCharString sessionDirectoryName; + SharedMemoryHelpers::VerifyStringOperation(id->AppendSessionDirectoryName(sessionDirectoryName)); + PathCharString sessionDirectoryPath; + SharedMemoryHelpers::BuildSessionDirectoryPath( + sessionDirectoryPath, + id, + true /* isLockFile */); + sessionDirectoryFileDescriptor = + SharedMemoryHelpers::EnsureDirectoryExists( + nullptr, + lockFilesDirectoryFileDescriptor, + sessionDirectoryName, + sessionDirectoryPath, + id, + false /* createIfNotExist */); + if (sessionDirectoryFileDescriptor != -1) + { + unlinkat(sessionDirectoryFileDescriptor, id->GetName(), 0); + SharedMemoryHelpers::CloseFile(sessionDirectoryFileDescriptor); + sessionDirectoryFileDescriptor = -1; + } + unlinkat(lockFilesDirectoryFileDescriptor, sessionDirectoryName, AT_REMOVEDIR); + SharedMemoryHelpers::CloseFile(lockFilesDirectoryFileDescriptor); + lockFilesDirectoryFileDescriptor = -1; + } } catch (SharedMemoryException) { // Ignore the error, just don't release shared data } + if (sessionDirectoryFileDescriptor != -1) + { + SharedMemoryHelpers::CloseFile(sessionDirectoryFileDescriptor); + } + if (lockFilesDirectoryFileDescriptor != -1) + { + SharedMemoryHelpers::CloseFile(lockFilesDirectoryFileDescriptor); + } #endif // !NAMED_MUTEX_USE_PTHREAD_MUTEX } diff --git a/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/namedmutex.cpp b/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/namedmutex.cpp index 6c62b7edf7354c..2841b72bfcdcde 100644 --- a/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/namedmutex.cpp +++ b/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/namedmutex.cpp @@ -19,6 +19,7 @@ const char *const ParentEventNamePrefix1 = "paltest_namedmutex_test1_pe1_"; const char *const ChildEventNamePrefix0 = "paltest_namedmutex_test1_ce0_"; const char *const ChildEventNamePrefix1 = "paltest_namedmutex_test1_ce1_"; const char *const ChildRunningEventNamePrefix = "paltest_namedmutex_test1_cr_"; +const char *const UserScopeNamePrefix = "paltest_namedmutex_test1_userscope_"; const char *const GlobalShmFilePathPrefix = "/tmp/.dotnet/shm/global/"; @@ -41,12 +42,18 @@ extern int (*test_sscanf)(const char *str, const char *format, ...); extern int(*test_close)(int fd); extern int (*test_unlink)(const char *pathname); extern unsigned int test_getpid(); +extern unsigned int test_geteuid(); extern int test_kill(unsigned int pid); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Test helpers extern bool TestFileExists(const char *path); +extern bool TestPathExists(const char *path); +extern bool TestCreateDirectory(const char *path); +extern void TestRemoveDirectory(const char *path); +extern bool TestPathStatus(const char *path, unsigned int expectedMode, unsigned int expectedUid); +extern bool TestChangeMode(const char *path, unsigned int mode); extern bool WriteHeaderInfo(const char *path, char sharedMemoryType, char version, int *fdRef); #define TestAssert(expression) \ @@ -184,6 +191,22 @@ HANDLE TestOpenMutex(const char *name) return OpenMutexA(SYNCHRONIZE, false, name); } +void TestCreateUserScopedMutex(AutoCloseMutexHandle &m, const char *name) +{ + m.Close(); + LPWSTR nameW = convert(name); + m = PAL_CreateMutexW(false, nameW, true, nullptr, 0); + free(nameW); +} + +HANDLE TestOpenUserScopedMutex(const char *name) +{ + LPWSTR nameW = convert(name); + HANDLE handle = PAL_OpenMutexW(nameW, true, nullptr, 0); + free(nameW); + return handle; +} + bool StartProcess(const char *funcName) { // Command line format: [stress] @@ -388,6 +411,133 @@ bool YieldToParent(AutoCloseMutexHandle parentEvents[2], AutoCloseMutexHandle ch //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Tests +bool ScopeTests() +{ +#if defined(__linux__) + const char *testName = "ScopeTests"; + char homeDirectory[MaxPathSize]; + test_snprintf(homeDirectory, MaxPathSize, "paltest_namedmutex_userscope_home_%u", g_parentPid); + TestRemoveDirectory(homeDirectory); + TestAssert(TestCreateDirectory(homeDirectory)); + TestAssert(SetEnvironmentVariableA("XDG_RUNTIME_DIR", "")); + TestAssert(SetEnvironmentVariableA("HOME", homeDirectory)); + + char persistedDotnetDirectory[MaxPathSize]; + char persistedCorefxDirectory[MaxPathSize]; + char persistedRuntimeDirectory[MaxPathSize]; + char basePath[MaxPathSize]; + char runtimeRoot[MaxPathSize]; + char sharedMemoryDirectory[MaxPathSize]; + char sessionDirectory[MaxPathSize]; + test_snprintf(persistedDotnetDirectory, MaxPathSize, "%s/.dotnet", homeDirectory); + test_snprintf(persistedCorefxDirectory, MaxPathSize, "%s/corefx", persistedDotnetDirectory); + test_snprintf(persistedRuntimeDirectory, MaxPathSize, "%s/Runtime", persistedCorefxDirectory); + test_snprintf(basePath, MaxPathSize, "%s/SharedMemory", persistedRuntimeDirectory); + test_snprintf(runtimeRoot, MaxPathSize, "%s/.dotnet", basePath); + test_snprintf(sharedMemoryDirectory, MaxPathSize, "%s/shm", runtimeRoot); + test_snprintf(sessionDirectory, MaxPathSize, "%s/global", sharedMemoryDirectory); + TestAssert(!TestPathExists(persistedDotnetDirectory)); + + char fileName[MaxPathSize]; + BuildName(testName, fileName, UserScopeNamePrefix); + char name[MaxPathSize]; + test_snprintf(name, MaxPathSize, "%s%s", GlobalPrefix, fileName); + + AutoCloseMutexHandle userMutex; + TestCreateUserScopedMutex(userMutex, name); + TestAssert(userMutex != nullptr); + TestAssert(GetLastError() != ERROR_ALREADY_EXISTS); + TestAssert(AutoCloseMutexHandle(TestOpenUserScopedMutex(name)) != nullptr); + TestAssert(AutoCloseMutexHandle(TestOpenMutex(name)) == nullptr); + + AutoCloseMutexHandle globalMutex; + TestCreateMutex(globalMutex, name); + TestAssert(globalMutex != nullptr); + TestAssert(GetLastError() != ERROR_ALREADY_EXISTS); + TestAssert(AutoCloseMutexHandle(TestOpenMutex(name)) != nullptr); + + AutoCloseMutexHandle duplicateUserMutex; + TestCreateUserScopedMutex(duplicateUserMutex, name); + TestAssert(duplicateUserMutex != nullptr); + TestAssert(GetLastError() == ERROR_ALREADY_EXISTS); + + char filePath[MaxPathSize]; + test_snprintf( + filePath, + MaxPathSize, + "%s/%s", + sessionDirectory, + fileName); + + unsigned int uid = test_geteuid(); + TestAssert(TestPathStatus(persistedDotnetDirectory, 0700, uid)); + TestAssert(TestPathStatus(persistedCorefxDirectory, 0700, uid)); + TestAssert(TestPathStatus(persistedRuntimeDirectory, 0700, uid)); + TestAssert(TestPathStatus(basePath, 0700, uid)); + TestAssert(TestPathStatus(runtimeRoot, 0700, uid)); + TestAssert(TestPathStatus(sharedMemoryDirectory, 0700, uid)); + TestAssert(TestPathStatus(sessionDirectory, 0700, uid)); + TestAssert(TestPathStatus(filePath, 0600, uid)); + + char invalidFileName[MaxPathSize]; + test_snprintf(invalidFileName, MaxPathSize, "%s_invalid_%u", UserScopeNamePrefix, g_parentPid); + char invalidName[MaxPathSize]; + test_snprintf(invalidName, MaxPathSize, "%s%s", GlobalPrefix, invalidFileName); + char invalidFilePath[MaxPathSize]; + test_snprintf(invalidFilePath, MaxPathSize, "%s/%s", sessionDirectory, invalidFileName); + int invalidFileDescriptor; + TestAssert(WriteHeaderInfo(invalidFilePath, 0, 1, &invalidFileDescriptor)); + TestAssert(TestChangeMode(invalidFilePath, 0666)); + TestAssert(AutoCloseMutexHandle(TestOpenUserScopedMutex(invalidName)) == nullptr); + TestAssert(test_close(invalidFileDescriptor) == 0); + TestAssert(test_unlink(invalidFilePath) == 0); + + char alternateBasePath[MaxPathSize]; + test_snprintf(alternateBasePath, MaxPathSize, "paltest_namedmutex_userscope_alternate_%u", g_parentPid); + TestRemoveDirectory(alternateBasePath); + TestAssert(TestCreateDirectory(alternateBasePath)); + TestAssert(SetEnvironmentVariableA("XDG_RUNTIME_DIR", alternateBasePath)); + + char cachedFileName[MaxPathSize]; + test_snprintf(cachedFileName, MaxPathSize, "%s_cached_%u", UserScopeNamePrefix, g_parentPid); + char cachedName[MaxPathSize]; + test_snprintf(cachedName, MaxPathSize, "%s%s", GlobalPrefix, cachedFileName); + AutoCloseMutexHandle cachedBaseUserMutex; + TestCreateUserScopedMutex(cachedBaseUserMutex, cachedName); + TestAssert(cachedBaseUserMutex != nullptr); + TestAssert(GetLastError() != ERROR_ALREADY_EXISTS); + + char cachedFilePath[MaxPathSize]; + test_snprintf(cachedFilePath, MaxPathSize, "%s/%s", sessionDirectory, cachedFileName); + TestAssert(TestFileExists(cachedFilePath)); + char alternateRuntimeRoot[MaxPathSize]; + test_snprintf(alternateRuntimeRoot, MaxPathSize, "%s/.dotnet", alternateBasePath); + TestAssert(!TestPathExists(alternateRuntimeRoot)); + + cachedBaseUserMutex.Close(); + duplicateUserMutex.Close(); + userMutex.Close(); + globalMutex.Close(); + + char lockFilesDirectory[MaxPathSize]; + char lockFilesSessionDirectory[MaxPathSize]; + test_snprintf(lockFilesDirectory, MaxPathSize, "%s/lockfiles", runtimeRoot); + test_snprintf(lockFilesSessionDirectory, MaxPathSize, "%s/global", lockFilesDirectory); + TestRemoveDirectory(lockFilesSessionDirectory); + TestRemoveDirectory(lockFilesDirectory); + TestRemoveDirectory(sessionDirectory); + TestRemoveDirectory(sharedMemoryDirectory); + TestRemoveDirectory(runtimeRoot); + TestRemoveDirectory(basePath); + TestRemoveDirectory(persistedRuntimeDirectory); + TestRemoveDirectory(persistedCorefxDirectory); + TestRemoveDirectory(persistedDotnetDirectory); + TestRemoveDirectory(homeDirectory); + TestRemoveDirectory(alternateBasePath); +#endif + return true; +} + bool NameTests() { const char *testName = "NameTests"; @@ -1100,6 +1250,7 @@ bool LockAndCloseWithoutThreadExitTests() bool (*const TestList[])() = { + ScopeTests, NameTests, HeaderMismatchTests, MutualExclusionTests, diff --git a/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/nopal.cpp b/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/nopal.cpp index 77665fe69c406d..01f1e23cf456a9 100644 --- a/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/nopal.cpp +++ b/src/coreclr/pal/tests/palsuite/threading/NamedMutex/test1/nopal.cpp @@ -27,11 +27,46 @@ unsigned int test_getpid() return getpid(); } +unsigned int test_geteuid() +{ + return geteuid(); +} + int test_kill(unsigned int pid) { return kill(pid, SIGKILL); } +bool TestCreateDirectory(const char *path) +{ + return mkdir(path, S_IRUSR | S_IWUSR | S_IXUSR) == 0; +} + +void TestRemoveDirectory(const char *path) +{ + rmdir(path); +} + +bool TestPathStatus(const char *path, unsigned int expectedMode, unsigned int expectedUid) +{ + struct stat statInfo; + return + lstat(path, &statInfo) == 0 && + (statInfo.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == expectedMode && + statInfo.st_uid == expectedUid; +} + +bool TestPathExists(const char *path) +{ + struct stat statInfo; + return lstat(path, &statInfo) == 0; +} + +bool TestChangeMode(const char *path, unsigned int mode) +{ + return chmod(path, mode) == 0; +} + bool TestFileExists(const char *path) { int fd = open(path, O_RDWR); diff --git a/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Proxy.cs b/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Proxy.cs index bb664f91298560..fb2d8083f3f9d2 100644 --- a/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Proxy.cs +++ b/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Proxy.cs @@ -722,6 +722,112 @@ public static IEnumerable CredentialsForProxy() } } + [Fact] + public async Task ProxiedRequest_HttpToHttpsRedirect_ProxyAuthNotSentToServer() + { + if (IsWinHttpHandler) + { + return; // Test targets SocketsHttpHandler proxy behavior. + } + + const string Content = "Hello world"; + var proxyCreds = new NetworkCredential("username", "password"); + + // Normal HTTP proxy that requires Basic authentication. + var proxyOptions = new LoopbackProxyServer.Options { AuthenticationSchemes = AuthenticationSchemes.Basic }; + using LoopbackProxyServer proxyServer = LoopbackProxyServer.Create(proxyOptions); + + // HTTPS server that ultimately handles the request (reached via a CONNECT tunnel). + var httpsOptions = new LoopbackServer.Options { UseSsl = true }; + await LoopbackServer.CreateServerAsync(async (httpsServer, httpsUri) => + { + // Plain HTTP server that returns a redirect to the HTTPS server (reached via plain forwarding). + await LoopbackServer.CreateServerAsync(async (httpServer, httpUri) => + { + using HttpClientHandler handler = CreateHttpClientHandler(allowAllCertificates: true); + handler.Proxy = new WebProxy(proxyServer.Uri) { Credentials = proxyCreds }; + using HttpClient client = CreateHttpClient(handler); + + // Client sends the request to the HTTP server through the HTTP proxy. + Task clientTask = client.GetAsync(httpUri); + + // The HTTP server responds with a redirect pointing at the HTTPS server. + await httpServer.AcceptConnectionSendResponseAndCloseAsync( + HttpStatusCode.Redirect, + additionalHeaders: $"Location: {httpsUri.AbsoluteUri}\r\n"); + + // The HTTPS server handles the redirected request and captures the headers it received. + List serverRequestHeaders = null; + await httpsServer.AcceptConnectionAsync(async connection => + { + serverRequestHeaders = await connection.ReadRequestHeaderAndSendResponseAsync(content: Content); + }); + + using HttpResponseMessage response = await clientTask; + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Content, await response.Content.ReadAsStringAsync()); + + // The credentials used to authenticate to the proxy must never be forwarded to the origin server. + Assert.DoesNotContain(serverRequestHeaders, h => h.StartsWith("Proxy-Authorization", StringComparison.OrdinalIgnoreCase)); + }, new LoopbackServer.Options()); + }, httpsOptions); + } + + [ConditionalFact(typeof(HttpClientHandler_Proxy_Test), nameof(IsSocketsHttpHandlerAndRemoteExecutorSupported))] + public async Task ProxiedRequest_HttpToNoProxyRedirect_ProxyAuthNotSentToServer() + { + await RemoteExecutor.Invoke(async (useVersionString) => + { + const string Content = "Hello world"; + var proxyCreds = new NetworkCredential("username", "password"); + + // Normal HTTP proxy that requires Basic authentication. + var proxyOptions = new LoopbackProxyServer.Options { AuthenticationSchemes = AuthenticationSchemes.Basic }; + using LoopbackProxyServer proxyServer = LoopbackProxyServer.Create(proxyOptions); + + // HTTP server that ultimately handles the request. + await LoopbackServer.CreateServerAsync(async (httpServer2, httpUri2) => + { + // Plain HTTP server that returns a redirect to the HTTPS server (reached via plain forwarding). + await LoopbackServer.CreateServerAsync(async (httpServer1, httpUri1) => + { + Console.WriteLine($"{proxyServer.Uri}, {httpUri1}, {httpUri2}"); + Environment.SetEnvironmentVariable("http_proxy", proxyServer.Uri.AbsoluteUri.ToString()); + Environment.SetEnvironmentVariable("no_proxy", "localhost"); + + using HttpClientHandler handler = CreateHttpClientHandler(allowAllCertificates: true); + handler.DefaultProxyCredentials = proxyCreds; + using HttpClient client = CreateHttpClient(handler, useVersionString); + + // Client sends the request to the HTTP server through the HTTP proxy. + Task clientTask = client.GetAsync(httpUri1); + + // The HTTP server responds with a redirect pointing at the HTTPS server. + await httpServer1.AcceptConnectionSendResponseAndCloseAsync( + HttpStatusCode.Redirect, + additionalHeaders: $"Location: {httpUri2.Scheme}://localhost:{httpUri2.Port}\r\n"); + + // The HTTPS server handles the redirected request and captures the headers it received. + List serverRequestHeaders = null; + await httpServer2.AcceptConnectionAsync(async connection => + { + serverRequestHeaders = await connection.ReadRequestHeaderAndSendResponseAsync(content: Content); + }); + + using HttpResponseMessage response = await clientTask; + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Content, await response.Content.ReadAsStringAsync()); + + Assert.Equal(2, proxyServer.Requests.Count); + Assert.NotNull(proxyServer.Requests[1].AuthorizationHeaderValueToken); + + // The credentials used to authenticate to the proxy must never be forwarded to the origin server. + Assert.DoesNotContain(serverRequestHeaders, h => h.StartsWith("Proxy-Authorization", StringComparison.OrdinalIgnoreCase)); + }, new LoopbackServer.Options()); + }); + }, UseVersion.ToString()).DisposeAsync(); + } + private sealed class TrackDisposalProxy : IWebProxy, IDisposable { public bool Disposed; diff --git a/src/libraries/Common/tests/System/Net/Http/LoopbackProxyServer.cs b/src/libraries/Common/tests/System/Net/Http/LoopbackProxyServer.cs index 0f4999b703497d..2eae9c5aaf5b2c 100644 --- a/src/libraries/Common/tests/System/Net/Http/LoopbackProxyServer.cs +++ b/src/libraries/Common/tests/System/Net/Http/LoopbackProxyServer.cs @@ -206,7 +206,9 @@ private async Task ProcessRequest(Socket clientSocket, StreamReader reader requestMessage.Headers.Add("Via", ViaHeaderValue); } - var handler = new HttpClientHandler() { UseProxy = false }; + // A forwarding proxy relays responses (including 3xx redirects) back to the client + // instead of following them itself, so disable auto-redirect on the outbound client. + var handler = new HttpClientHandler() { UseProxy = false, AllowAutoRedirect = false }; using (HttpClient outboundClient = new HttpClient(handler)) using (HttpResponseMessage response = await outboundClient.SendAsync(requestMessage).ConfigureAwait(false)) { diff --git a/src/libraries/System.Formats.Cbor/src/Resources/Strings.resx b/src/libraries/System.Formats.Cbor/src/Resources/Strings.resx index 4fde1bed0fd72c..8b9481398dda74 100644 --- a/src/libraries/System.Formats.Cbor/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Cbor/src/Resources/Strings.resx @@ -219,6 +219,9 @@ Not a valid tagged decimal encoding. + + The depth of the CBOR data exceeded the maximum allowed depth of {0}. + Not a valid CBOR value encoding. diff --git a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Array.cs b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Array.cs index 261ad78dea3056..0416b09ce07a28 100644 --- a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Array.cs +++ b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Array.cs @@ -16,6 +16,7 @@ public partial class CborReader public int? ReadStartArray() { CborInitialByte header = PeekInitialByte(expectedType: CborMajorType.Array); + EnsureMaxDepthNotExceeded(); if (header.AdditionalInfo == CborAdditionalInfo.IndefiniteLength) { diff --git a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Map.cs b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Map.cs index 2de5c024853a95..1ca66a0537015a 100644 --- a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Map.cs +++ b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.Map.cs @@ -31,6 +31,7 @@ public partial class CborReader { int? length; CborInitialByte header = PeekInitialByte(expectedType: CborMajorType.Map); + EnsureMaxDepthNotExceeded(); if (header.AdditionalInfo == CborAdditionalInfo.IndefiniteLength) { diff --git a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.String.cs b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.String.cs index 284619e83d3d0d..a5d7821966fc52 100644 --- a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.String.cs +++ b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.String.cs @@ -132,6 +132,7 @@ public ReadOnlyMemory ReadDefiniteLengthByteString() public void ReadStartIndefiniteLengthByteString() { CborInitialByte header = PeekInitialByte(expectedType: CborMajorType.ByteString); + EnsureMaxDepthNotExceeded(); if (header.AdditionalInfo != CborAdditionalInfo.IndefiniteLength) { @@ -299,6 +300,7 @@ public ReadOnlyMemory ReadDefiniteLengthTextStringBytes() public void ReadStartIndefiniteLengthTextString() { CborInitialByte header = PeekInitialByte(expectedType: CborMajorType.TextString); + EnsureMaxDepthNotExceeded(); if (header.AdditionalInfo != CborAdditionalInfo.IndefiniteLength) { @@ -445,6 +447,8 @@ private bool TryReadIndefiniteLengthTextStringConcatenated(Span destinatio // containing the individual chunk payloads private List<(int Offset, int Length)> ReadIndefiniteLengthStringChunkRanges(CborMajorType type, out int encodingLength, out int concatenatedBufferSize) { + EnsureMaxDepthNotExceeded(); + List<(int Offset, int Length)> ranges = AcquireIndefiniteLengthStringRangeList(); ReadOnlySpan data = GetRemainingBytes(); concatenatedBufferSize = 0; diff --git a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.cs b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.cs index 88fa5ab68d8982..e07ce3fa888956 100644 --- a/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.cs +++ b/src/libraries/System.Formats.Cbor/src/System/Formats/Cbor/Reader/CborReader.cs @@ -3,12 +3,17 @@ using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; namespace System.Formats.Cbor { /// A stateful, forward-only reader for Concise Binary Object Representation (CBOR) encoded data. public partial class CborReader { + private const int DefaultMaxDepth = 64; + private const string MaxDepthAppContextSwitchName = "System.Formats.Cbor.CborReader.MaxDepth"; + private static readonly int s_maxDepth = GetMaxDepth(); + private ReadOnlyMemory _data; private int _offset; @@ -171,8 +176,36 @@ private void ValidateNextByteIsBreakByte() } } + private static int GetMaxDepth() + { + object? configuredValue = +#if NETFRAMEWORK + AppDomain.CurrentDomain.GetData(MaxDepthAppContextSwitchName); +#else + AppContext.GetData(MaxDepthAppContextSwitchName); +#endif + + return configuredValue switch + { + int value when value >= 0 => value, + long value when value is >= 0 and <= int.MaxValue => (int)value, + string value when int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int maxDepth) && maxDepth >= 0 => maxDepth, + _ => DefaultMaxDepth, + }; + } + + private void EnsureMaxDepthNotExceeded() + { + if (CurrentDepth >= s_maxDepth) + { + throw new CborContentException(SR.Format(SR.Cbor_Reader_MaximumDepthExceeded, s_maxDepth)); + } + } + private void PushDataItem(CborMajorType majorType, int? definiteLength) { + Debug.Assert(CurrentDepth < s_maxDepth); + _nestedDataItems ??= new Stack(); var frame = new StackFrame( diff --git a/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.SkipValue.cs b/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.SkipValue.cs index 9431d4063445c1..2a72127c560d7a 100644 --- a/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.SkipValue.cs +++ b/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.SkipValue.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using Microsoft.DotNet.RemoteExecutor; using Test.Cryptography; using Xunit; @@ -259,21 +260,30 @@ public static void SkipToParent_RootContext_ShouldThrowInvalidOperationException Assert.Throws(() => reader.SkipToParent()); } - [Theory] - [InlineData(50_000)] - public static void SkipValue_ExtremelyNestedValues_ShouldNotStackOverflow(int depth) + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void SkipValue_ExtremelyNestedValues_ShouldNotStackOverflow() { - // Construct a valid CBOR encoding with extreme nesting: - // defines a tower of `depth` nested singleton arrays, - // with the innermost array containing zero. - byte[] encoding = new byte[depth + 1]; - encoding.AsSpan(0, depth).Fill(0x81); // array of length 1 - encoding[depth] = 0; - - var reader = new CborReader(encoding); - - reader.SkipValue(); - Assert.Equal(CborReaderState.Finished, reader.PeekState()); + RemoteExecutor.Invoke(static () => + { + const int Depth = 50_000; +#if NETFRAMEWORK + AppDomain.CurrentDomain.SetData(MaxDepthAppContextSwitchName, Depth); +#else + AppContext.SetData(MaxDepthAppContextSwitchName, Depth); +#endif + + // Construct a valid CBOR encoding with extreme nesting: + // defines a tower of `depth` nested singleton arrays, + // with the innermost array containing zero. + byte[] encoding = new byte[Depth + 1]; + encoding.AsSpan(0, Depth).Fill(0x81); // array of length 1 + encoding[Depth] = 0; + + var reader = new CborReader(encoding); + + reader.SkipValue(); + Assert.Equal(CborReaderState.Finished, reader.PeekState()); + }).Dispose(); } public static IEnumerable SkipTestInputs => SampleCborValues.Select(x => new [] { x }); diff --git a/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs b/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs index 9591cea9f5e808..05899d667a2c42 100644 --- a/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs +++ b/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; +using Microsoft.DotNet.RemoteExecutor; using Test.Cryptography; using Xunit; @@ -11,6 +12,8 @@ namespace System.Formats.Cbor.Tests { public partial class CborReaderTests { + private const string MaxDepthAppContextSwitchName = "System.Formats.Cbor.CborReader.MaxDepth"; + [Theory] [InlineData((CborConformanceMode)(-1))] public static void InvalidConformanceMode_ShouldThrowArgumentOutOfRangeException(CborConformanceMode mode) @@ -54,6 +57,95 @@ public static void Depth_ShouldReturnExpectedValue(int depth) Assert.Equal(CborReaderState.Finished, reader.PeekState()); } + [Fact] + public static void MaxDepth_DefaultValueIs64() + { + byte[] encoding = Enumerable.Repeat((byte)0x81, 65).Append((byte)0x01).ToArray(); + var reader = new CborReader(encoding); + + for (int i = 0; i < 64; i++) + { + reader.ReadStartArray(); + } + + int bytesRemaining = reader.BytesRemaining; + CborContentException exception = Assert.Throws(() => reader.ReadStartArray()); + Assert.Contains("maximum allowed depth of 64", exception.Message); + Assert.Equal(bytesRemaining, reader.BytesRemaining); + } + + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void MaxDepth_CanBeConfiguredUsingAppContext() + { + RemoteExecutor.Invoke(static () => + { +#if NETFRAMEWORK + AppDomain.CurrentDomain.SetData(MaxDepthAppContextSwitchName, "2"); +#else + AppContext.SetData(MaxDepthAppContextSwitchName, "2"); +#endif + + AssertDepthExceeded( + new byte[] { 0x81, 0x81, 0x81, 0x01 }, + static reader => + { + reader.ReadStartArray(); + reader.ReadStartArray(); + }, + static reader => reader.ReadStartArray()); + + AssertDepthExceeded( + new byte[] { 0xA1, 0x00, 0xA1, 0x00, 0xA1, 0x00, 0x01 }, + static reader => + { + reader.ReadStartMap(); + reader.ReadInt32(); + reader.ReadStartMap(); + reader.ReadInt32(); + }, + static reader => reader.ReadStartMap()); + + AssertDepthExceeded( + new byte[] { 0x81, 0x81, 0x5F, 0x41, 0x01, 0xFF }, + static reader => + { + reader.ReadStartArray(); + reader.ReadStartArray(); + }, + static reader => reader.ReadByteString()); + + AssertDepthExceeded( + new byte[] { 0x81, 0x81, 0x7F, 0x61, 0x61, 0xFF }, + static reader => + { + reader.ReadStartArray(); + reader.ReadStartArray(); + }, + static reader => reader.ReadTextString()); + + AssertDepthExceeded( + new byte[] { 0x81, 0x81, 0x81, 0x01 }, + static reader => { }, + static reader => reader.SkipValue()); + + AssertDepthExceeded( + new byte[] { 0x81, 0x81, 0x81, 0x01 }, + static reader => { }, + static reader => reader.ReadEncodedValue()); + + static void AssertDepthExceeded(byte[] encoding, Action setup, Action action) + { + var reader = new CborReader(encoding, CborConformanceMode.Lax); + setup(reader); + int bytesRemaining = reader.BytesRemaining; + + CborContentException exception = Assert.Throws(() => action(reader)); + Assert.Contains("maximum allowed depth of 2", exception.Message); + Assert.Equal(bytesRemaining, reader.BytesRemaining); + } + }).Dispose(); + } + [Fact] public static void Reset_DoesNotAffect_ConformanceMode() { diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index ac632535dfd180..f79328ac39d79a 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -205,4 +205,13 @@ Cannot write the unseekable data stream of entry '{0}' into an unseekable archive stream. + + The extended header contains invalid records. + + + The file appears to be a {0} archive. TAR format expected. + + + Too many levels of symbolic links in '{0}'. + diff --git a/src/libraries/System.Formats.Tar/src/System.Formats.Tar.csproj b/src/libraries/System.Formats.Tar/src/System.Formats.Tar.csproj index 6f34a34cc6e7de..1c1bee3ad2c39a 100644 --- a/src/libraries/System.Formats.Tar/src/System.Formats.Tar.csproj +++ b/src/libraries/System.Formats.Tar/src/System.Formats.Tar.csproj @@ -45,6 +45,7 @@ + @@ -52,6 +53,13 @@ + + + + + + + diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs index 6441b4c3b98368..9f13dd181c7562 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs @@ -204,7 +204,8 @@ public void ExtractToFile(string destinationFileName, bool overwrite) { throw new InvalidOperationException(SR.Format(SR.TarEntryTypeNotSupportedForExtracting, EntryType)); } - ExtractToFileInternal(destinationFileName, linkTargetPath: null, overwrite); + + ExtractToFileInternal(destinationFileName, linkTargetPath: null, overwrite, pendingLinks: null); } /// @@ -238,7 +239,8 @@ public Task ExtractToFileAsync(string destinationFileName, bool overwrite, Cance { return Task.FromException(new InvalidOperationException(SR.Format(SR.TarEntryTypeNotSupportedForExtracting, EntryType))); } - return ExtractToFileInternalAsync(destinationFileName, linkTargetPath: null, overwrite, cancellationToken); + + return ExtractToFileInternalAsync(destinationFileName, linkTargetPath: null, overwrite, pendingLinks: null, cancellationToken); } /// @@ -300,49 +302,68 @@ public Stream? DataStream internal abstract bool IsDataStreamSetterSupported(); // Extracts the current entry to a location relative to the specified directory. - internal void ExtractRelativeToDirectory(string destinationDirectoryPath, bool overwrite, SortedDictionary? pendingModes, Stack<(string, DateTimeOffset)> directoryModificationTimes) + internal void ExtractRelativeToDirectory(string destinationDirectoryPath, bool overwrite, ref TarHelpers.ExtractState extractState) { - (string destinationFullPath, string? linkTargetPath) = GetDestinationAndLinkPaths(destinationDirectoryPath); + (string destinationFullPath, string? linkTargetPath, bool deferred) = GetDestinationAndLinkPaths(destinationDirectoryPath); + + if (deferred) + { + // Create a placeholder file for the deferred symlink. + Debug.Assert(EntryType is TarEntryType.SymbolicLink); + TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, extractState.PendingModes); + CreateSymlinkPlaceholderFile(destinationFullPath, linkTargetPath!, overwrite, extractState.GetOrCreatePendingLinks()); + return; + } if (EntryType == TarEntryType.Directory) { - TarHelpers.CreateDirectory(destinationFullPath, Mode, pendingModes); - TarHelpers.UpdatePendingModificationTimes(directoryModificationTimes, destinationFullPath, ModificationTime); + TarHelpers.CreateDirectory(destinationFullPath, Mode, extractState.PendingModes); + TarHelpers.UpdatePendingModificationTimes(extractState.DirectoryModificationTimes, destinationFullPath, ModificationTime); } else { // If it is a file, create containing directory. - TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, pendingModes); - ExtractToFileInternal(destinationFullPath, linkTargetPath, overwrite); + TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, extractState.PendingModes); + ExtractToFileInternal(destinationFullPath, linkTargetPath, overwrite, extractState.PendingLinks); } } // Asynchronously extracts the current entry to a location relative to the specified directory. - internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, bool overwrite, SortedDictionary? pendingModes, Stack<(string, DateTimeOffset)> directoryModificationTimes, CancellationToken cancellationToken) + internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, bool overwrite, ref TarHelpers.ExtractState extractState, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return Task.FromCanceled(cancellationToken); } - (string destinationFullPath, string? linkTargetPath) = GetDestinationAndLinkPaths(destinationDirectoryPath); + (string destinationFullPath, string? linkTargetPath, bool deferred) = GetDestinationAndLinkPaths(destinationDirectoryPath); + + if (deferred) + { + // Create a placeholder file for the deferred symlink. + Debug.Assert(EntryType is TarEntryType.SymbolicLink); + TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, extractState.PendingModes); + CreateSymlinkPlaceholderFile(destinationFullPath, linkTargetPath!, overwrite, extractState.GetOrCreatePendingLinks()); + return Task.CompletedTask; + } if (EntryType == TarEntryType.Directory) { - TarHelpers.CreateDirectory(destinationFullPath, Mode, pendingModes); - TarHelpers.UpdatePendingModificationTimes(directoryModificationTimes, destinationFullPath, ModificationTime); + TarHelpers.CreateDirectory(destinationFullPath, Mode, extractState.PendingModes); + TarHelpers.UpdatePendingModificationTimes(extractState.DirectoryModificationTimes, destinationFullPath, ModificationTime); return Task.CompletedTask; } else { // If it is a file, create containing directory. - TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, pendingModes); - return ExtractToFileInternalAsync(destinationFullPath, linkTargetPath, overwrite, cancellationToken); + TarHelpers.CreateDirectory(Path.GetDirectoryName(destinationFullPath)!, mode: null, extractState.PendingModes); + return ExtractToFileInternalAsync(destinationFullPath, linkTargetPath, overwrite, extractState.PendingLinks, cancellationToken); } } // Gets the sanitized paths for the file destination and link target paths to be used when extracting relative to a directory. - private (string, string?) GetDestinationAndLinkPaths(string destinationDirectoryPath) + // Returns deferred=true when a symlink is deferred (placeholder file created, real link created after all entries are extracted). + private (string DestinationFullPath, string? LinkTargetPath, bool Deferred) GetDestinationAndLinkPaths(string destinationDirectoryPath) { Debug.Assert(!string.IsNullOrEmpty(destinationDirectoryPath)); Debug.Assert(Path.IsPathFullyQualified(destinationDirectoryPath)); @@ -351,7 +372,7 @@ internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, b string? fileDestinationPath = GetFullDestinationPath( destinationDirectoryPath, Path.IsPathFullyQualified(name) ? name : Path.Join(destinationDirectoryPath, name)); - if (fileDestinationPath is null || FilePathEscapesDirectory(destinationDirectoryPath, fileDestinationPath)) + if (fileDestinationPath is null) { throw new IOException(SR.Format(SR.TarExtractingResultsFileOutside, name, destinationDirectoryPath)); } @@ -369,13 +390,17 @@ internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, b { throw new IOException(SR.Format(SR.TarExtractingResultsLinkOutside, linkName, destinationDirectoryPath)); } - string? linkDestination = GetFullDestinationPath( - destinationDirectoryPath, - Path.IsPathFullyQualified(linkName) ? linkName : Path.Join(Path.GetDirectoryName(fileDestinationPath), linkName)); - if (linkDestination is null || FilePathEscapesDirectory(destinationDirectoryPath, linkDestination)) + + if (!LinkDestinationIsWithinDirectory(destinationDirectoryPath, fileDestinationPath, linkName, out bool deferred)) { throw new IOException(SR.Format(SR.TarExtractingResultsLinkOutside, linkName, destinationDirectoryPath)); } + + if (deferred) + { + return (fileDestinationPath, linkName, true); + } + // Use the linkName for creating the symbolic link. linkTargetPath = linkName; } @@ -387,7 +412,7 @@ internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, b string? linkDestination = GetFullDestinationPath( destinationDirectoryPath, Path.Join(destinationDirectoryPath, linkName)); - if (linkDestination is null || FilePathEscapesDirectory(destinationDirectoryPath, linkDestination)) + if (linkDestination is null) { throw new IOException(SR.Format(SR.TarExtractingResultsLinkOutside, linkName, destinationDirectoryPath)); } @@ -395,118 +420,71 @@ internal Task ExtractRelativeToDirectoryAsync(string destinationDirectoryPath, b linkTargetPath = linkDestination; } - return (fileDestinationPath, linkTargetPath); + return (fileDestinationPath, linkTargetPath, false); } // Prevent an archive from escaping the extraction root through symlinks that were created by earlier entries in the same archive. // This protection applies only to links introduced by the archive itself. It is not intended to defend against preexisting symlinks // already present on disk before extraction. - private static bool FilePathEscapesDirectory(string destinationDirectoryPath, string fileDestinationPath) + private static bool LinkDestinationIsWithinDirectory(string destinationDirectoryPath, string linkFullPath, string linkTarget, out bool deferred) { - // Windows is case insensitive while Linux is case sensitive - // This ensures the comparison is consistent with how the OS would resolve the paths - StringComparison pathComparison = OperatingSystem.IsWindows() - ? StringComparison.OrdinalIgnoreCase - : StringComparison.Ordinal; - - string resolvedDest = ResolvePhysicalPath(destinationDirectoryPath); - - // Use the logical destination path for computing the relative path - string logicalDest = Path.GetFullPath(destinationDirectoryPath); - string logicalPrefix = logicalDest.EndsWith(Path.DirectorySeparatorChar) - ? logicalDest - : logicalDest + Path.DirectorySeparatorChar; + // Symbolic links with ".." segments are tricky as they resolve against the physical location of the link and not the lexical path used to access it (which may contain symlinks to "."). + // We defer the creation of such links until after all entries are extracted, at which point we can properly validate the destination. + deferred = HasDotDotSegment(linkTarget); - string destPrefix = resolvedDest.EndsWith(Path.DirectorySeparatorChar) - ? resolvedDest - : resolvedDest + Path.DirectorySeparatorChar; - - // Normalize file path (resolves .. and . but not symlinks) - string normalizedFile = Path.GetFullPath(fileDestinationPath); - - // Guard with StartsWith before computing relative path - if (!normalizedFile.StartsWith(logicalPrefix, pathComparison) && - !normalizedFile.Equals(logicalDest, pathComparison)) + // Require the target path to lexically reside within the destination. + if (deferred || Path.IsPathFullyQualified(linkTarget)) { - return true; - } - - // Walk relative components, resolving symlinks at each step. - // When the file resolves to the destination directory itself, there are no components to walk, - // so the relative path is empty. Guarding here avoids an out-of-range Substring on the prefix length. - string relative = normalizedFile.Equals(logicalDest, pathComparison) - ? string.Empty - : normalizedFile.Substring(logicalPrefix.Length) - .TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - - string[] components = relative.Split(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, - StringSplitOptions.RemoveEmptyEntries); - - string current = resolvedDest; - - foreach (string component in components) - { - current = Path.Combine(current, component); - current = ResolveSymlink(current); - - string normalizedCurrent = Path.GetFullPath(current); - if (!normalizedCurrent.StartsWith(destPrefix, pathComparison) && - !normalizedCurrent.Equals(resolvedDest, pathComparison)) + string resolvedPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(linkFullPath)!, linkTarget)); + if (!TarHelpers.FullPathIsEqualOrNested(destinationDirectoryPath, resolvedPath)) { - return true; + return false; } } - return false; + return true; } - private static string ResolveSymlink(string path) + private static bool HasDotDotSegment(ReadOnlySpan path) { - var info = new FileInfo(path); - - // Check LinkTarget first so dangling symlinks/junctions (whose final target doesn't exist yet) - // are still resolved to their raw target, rather than being treated as a non-link. - if (info.LinkTarget is null) + foreach (Range range in path.SplitAny(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) { - return Path.GetFullPath(path); + if (path[range] is "..") + { + return true; + } } - FileSystemInfo target = info.ResolveLinkTarget(returnFinalTarget: true) ?? info; - return target.FullName; + return false; } - // Resolves the full path of the specified path, resolving symlinks at each step. - // This is needed to mitigate malicious entries in the archive that could lead to writing files outside of the intended directory. - private static string ResolvePhysicalPath(string path) + private static void CreateSymlinkPlaceholderFile(string filePath, string linkTarget, bool overwrite, Dictionary pendingLinks) { - string fullPath = Path.GetFullPath(path); - string? root = Path.GetPathRoot(fullPath); + // Performs the same existing-destination handling as the non-deferred path, which includes + // dropping the pending entry of a placeholder that is being overwritten. + VerifyDestinationPath(filePath, overwrite, pendingLinks); - if (root is null) + var options = new FileStreamOptions { - return fullPath; - } - - string[] components = fullPath.Substring(root.Length) - .Split(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); - string current = root; - foreach (string component in components) + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = 0, + }; + if (!OperatingSystem.IsWindows()) { - current = Path.Combine(current, component); - if (Path.Exists(current)) - { - current = ResolveSymlink(current); - } + options.UnixCreateMode = UnixFileMode.None; } - - return current; + new FileStream(filePath, options).Dispose(); + FileId fileId = TarHelpers.GetFileId(filePath); + pendingLinks[fileId] = (filePath, linkTarget); } // Returns the full destination path if the path is the destinationDirectory or a subpath. Otherwise, returns null. private static string? GetFullDestinationPath(string destinationDirectoryFullPath, string qualifiedPath) { Debug.Assert(Path.IsPathFullyQualified(qualifiedPath), $"{qualifiedPath} is not qualified"); - Debug.Assert(PathInternal.EndsInDirectorySeparator(destinationDirectoryFullPath), "caller must ensure the path ends with a separator."); + Debug.Assert(destinationDirectoryFullPath[^1] == Path.DirectorySeparatorChar, "caller must ensure the path ends with a separator."); string fullPath = Path.GetFullPath(qualifiedPath); // Removes relative segments @@ -514,9 +492,9 @@ private static string ResolvePhysicalPath(string path) } // Extracts the current entry into the filesystem, regardless of the entry type. - private void ExtractToFileInternal(string filePath, string? linkTargetPath, bool overwrite) + private void ExtractToFileInternal(string filePath, string? linkTargetPath, bool overwrite, Dictionary? pendingLinks) { - VerifyDestinationPath(filePath, overwrite); + VerifyDestinationPath(filePath, overwrite, pendingLinks); if (EntryType is TarEntryType.RegularFile or TarEntryType.V7RegularFile or TarEntryType.ContiguousFile) { @@ -529,13 +507,13 @@ private void ExtractToFileInternal(string filePath, string? linkTargetPath, bool } // Asynchronously extracts the current entry into the filesystem, regardless of the entry type. - private Task ExtractToFileInternalAsync(string filePath, string? linkTargetPath, bool overwrite, CancellationToken cancellationToken) + private Task ExtractToFileInternalAsync(string filePath, string? linkTargetPath, bool overwrite, Dictionary? pendingLinks, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return Task.FromCanceled(cancellationToken); } - VerifyDestinationPath(filePath, overwrite); + VerifyDestinationPath(filePath, overwrite, pendingLinks); if (EntryType is TarEntryType.RegularFile or TarEntryType.V7RegularFile or TarEntryType.ContiguousFile) { @@ -611,7 +589,7 @@ private void CreateNonRegularFile(string filePath, string? linkTargetPath) } // Verifies there's a writable destination. - private static void VerifyDestinationPath(string filePath, bool overwrite) + private static void VerifyDestinationPath(string filePath, bool overwrite, Dictionary? pendingLinks) { string? directoryPath = Path.GetDirectoryName(filePath); // If the destination contains a directory segment, need to check that it exists @@ -636,6 +614,15 @@ private static void VerifyDestinationPath(string filePath, bool overwrite) { throw new IOException(SR.Format(SR.IO_AlreadyExists_Name, filePath)); } + + // If this path is a deferred symlink placeholder, drop its pending entry so it is not + // materialized later and clobber the entry that is overwriting it. Use the no-follow + // file id (TryGetPathInfo) so we match the placeholder itself, not a symlink's target. + if (pendingLinks is { Count: > 0 } && TarHelpers.TryGetPathInfo(filePath, out FileId placeholderId, out _)) + { + pendingLinks.Remove(placeholderId); + } + File.Delete(filePath); } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarFile.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarFile.cs index f1d6a059c57350..254dee3e10925a 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarFile.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarFile.cs @@ -457,18 +457,16 @@ private static void ExtractToDirectoryInternal(Stream source, string destination using TarReader reader = new TarReader(source, leaveOpen); - SortedDictionary? pendingModes = TarHelpers.CreatePendingModesDictionary(); - var directoryModificationTimes = new Stack<(string, DateTimeOffset)>(); + var extractState = new TarHelpers.ExtractState(); TarEntry? entry; while ((entry = reader.GetNextEntry()) != null) { if (entry.EntryType is not TarEntryType.GlobalExtendedAttributes) { - entry.ExtractRelativeToDirectory(destinationDirectoryFullPath, overwriteFiles, pendingModes, directoryModificationTimes); + entry.ExtractRelativeToDirectory(destinationDirectoryFullPath, overwriteFiles, ref extractState); } } - TarHelpers.SetPendingModes(pendingModes); - TarHelpers.SetPendingModificationTimes(directoryModificationTimes); + extractState.Apply(destinationDirectoryFullPath); } // Asynchronously extracts the contents of a tar file into the specified directory. @@ -499,8 +497,8 @@ private static async Task ExtractToDirectoryInternalAsync(Stream source, string VerifyExtractToDirectoryArguments(source, destinationDirectoryFullPath); cancellationToken.ThrowIfCancellationRequested(); - SortedDictionary? pendingModes = TarHelpers.CreatePendingModesDictionary(); - var directoryModificationTimes = new Stack<(string, DateTimeOffset)>(); + var extractState = new TarHelpers.ExtractState(); + TarReader reader = new TarReader(source, leaveOpen); await using (reader.ConfigureAwait(false)) { @@ -509,12 +507,14 @@ private static async Task ExtractToDirectoryInternalAsync(Stream source, string { if (entry.EntryType is not TarEntryType.GlobalExtendedAttributes) { - await entry.ExtractRelativeToDirectoryAsync(destinationDirectoryFullPath, overwriteFiles, pendingModes, directoryModificationTimes, cancellationToken).ConfigureAwait(false); + // Pass extractState by ref in a synchronous statement; the ref is used only synchronously + // (a ref argument cannot be part of an awaited expression). + Task extractTask = entry.ExtractRelativeToDirectoryAsync(destinationDirectoryFullPath, overwriteFiles, ref extractState, cancellationToken); + await extractTask.ConfigureAwait(false); } } } - TarHelpers.SetPendingModes(pendingModes); - TarHelpers.SetPendingModificationTimes(directoryModificationTimes); + extractState.Apply(destinationDirectoryFullPath); } [Conditional("DEBUG")] diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Unix.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Unix.cs index 303d2f5d65771c..c0438072c0ee01 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Unix.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Unix.cs @@ -7,6 +7,22 @@ namespace System.Formats.Tar { + internal struct FileId : IEquatable + { + internal long Dev; + internal long Ino; + + internal FileId(long dev, long ino) + { + Dev = dev; + Ino = ino; + } + + public bool Equals(FileId other) => Dev == other.Dev && Ino == other.Ino; + public override bool Equals(object? obj) => obj is FileId other && Equals(other); + public override int GetHashCode() => HashCode.Combine(Dev, Ino); + } + internal static partial class TarHelpers { private static readonly Lazy s_umask = new Lazy(DetermineUMask); @@ -114,5 +130,30 @@ internal static void SetPendingModes(SortedDictionary? pen File.SetUnixFileMode(dir.Key, dir.Value & ~umask); } } + + internal static FileId GetFileId(string filePath) + { + Interop.CheckIo(Interop.Sys.Stat(filePath, out Interop.Sys.FileStatus status), filePath); + return new FileId(status.Dev, status.Ino); + } + + internal static bool TryGetPathInfo(string path, out FileId fileId, out bool isSymlink) + { + int rv = Interop.Sys.LStat(path, out Interop.Sys.FileStatus status); + if (rv != 0) + { + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (errorInfo.Error == Interop.Error.ENOENT || errorInfo.Error == Interop.Error.ENOTDIR) + { + fileId = default; + isSymlink = false; + return false; + } + throw Interop.GetExceptionForIoErrno(errorInfo, path); + } + fileId = new FileId(status.Dev, status.Ino); + isSymlink = (status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFLNK; + return true; + } } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Windows.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Windows.cs index 90a3c22bff6ea9..b3f9d3f75bc1bb 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Windows.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.Windows.cs @@ -4,10 +4,27 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Text; +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; namespace System.Formats.Tar { + internal struct FileId : IEquatable + { + internal long VolumeSerialNumber; + internal long FileIndex; + + internal FileId(long volumeSerialNumber, long fileIndex) + { + VolumeSerialNumber = volumeSerialNumber; + FileIndex = fileIndex; + } + + public bool Equals(FileId other) => VolumeSerialNumber == other.VolumeSerialNumber && FileIndex == other.FileIndex; + public override bool Equals(object? obj) => obj is FileId other && Equals(other); + public override int GetHashCode() => HashCode.Combine(VolumeSerialNumber, FileIndex); + } + internal static partial class TarHelpers { internal static SortedDictionary? CreatePendingModesDictionary() @@ -20,5 +37,49 @@ internal static void CreateDirectory(string fullPath, UnixFileMode? mode, Sorted internal static void SetPendingModes(SortedDictionary? pendingModes) => Debug.Assert(pendingModes is null); + + internal static FileId GetFileId(string filePath) + { + using SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + if (!Interop.Kernel32.GetFileInformationByHandle(handle, out Interop.Kernel32.BY_HANDLE_FILE_INFORMATION fileInfo)) + { + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError(), filePath); + } + ulong fileIndex = ((ulong)fileInfo.nFileIndexHigh << 32) | fileInfo.nFileIndexLow; + return new FileId((long)fileInfo.dwVolumeSerialNumber, (long)fileIndex); + } + + internal static bool TryGetPathInfo(string path, out FileId fileId, out bool isSymlink) + { + using SafeFileHandle handle = Interop.Kernel32.CreateFile( + path, + Interop.Kernel32.GenericOperations.GENERIC_READ, + FileShare.ReadWrite | FileShare.Delete, + FileMode.Open, + Interop.Kernel32.FileOperations.FILE_FLAG_OPEN_REPARSE_POINT | Interop.Kernel32.FileOperations.FILE_FLAG_BACKUP_SEMANTICS); + + if (handle.IsInvalid) + { + int error = Marshal.GetLastPInvokeError(); + if (error == Interop.Errors.ERROR_FILE_NOT_FOUND || error == Interop.Errors.ERROR_PATH_NOT_FOUND) + { + fileId = default; + isSymlink = false; + return false; + } + throw Win32Marshal.GetExceptionForWin32Error(error, path); + } + + if (!Interop.Kernel32.GetFileInformationByHandle(handle, out Interop.Kernel32.BY_HANDLE_FILE_INFORMATION fileInfo)) + { + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError(), path); + } + + ulong fileIndex = ((ulong)fileInfo.nFileIndexHigh << 32) | fileInfo.nFileIndexLow; + fileId = new FileId((long)fileInfo.dwVolumeSerialNumber, (long)fileIndex); + const uint FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400; + isSymlink = (fileInfo.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; + return true; + } } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs index a2e037e615e730..a5f075cbe420b8 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; @@ -17,9 +18,35 @@ namespace System.Formats.Tar // Static class containing a variety of helper methods. internal static partial class TarHelpers { + private const int MaxSymlinkDepth = 40; + internal const short RecordSize = 512; internal const int MaxBufferLength = 4096; + internal struct ExtractState + { + internal SortedDictionary? PendingModes; + internal Stack<(string, DateTimeOffset)> DirectoryModificationTimes; + // Allocated lazily on the first deferred symlink; most archives have none. + internal Dictionary? PendingLinks; + + public ExtractState() + { + PendingModes = CreatePendingModesDictionary(); + DirectoryModificationTimes = new(); + } + + internal Dictionary GetOrCreatePendingLinks() + => PendingLinks ??= new(); + + internal readonly void Apply(string destinationDirectoryPath) + { + CreatePendingLinks(destinationDirectoryPath, PendingLinks); + SetPendingModes(PendingModes); + SetPendingModificationTimes(DirectoryModificationTimes); + } + } + internal const UnixFileMode ValidUnixFileModes = UnixFileMode.UserRead | UnixFileMode.UserWrite | @@ -480,5 +507,297 @@ private static void AttemptDirectorySetLastWriteTime(string fullPath, DateTimeOf // Some OSes like Android might not support setting the last write time, the extraction should not fail because of that } } + + // We delayed creating some symbolic links because they might target locations outside the destination. + // Any symlink that is in the destination directory at the time this method is called is either one that points in the destination + // or a pre-existing symlink that we are allowed to follow outside. + internal static void CreatePendingLinks(string destinationDirectoryFullPath, Dictionary? pendingLinks) + { + Debug.Assert(destinationDirectoryFullPath[^1] == Path.DirectorySeparatorChar); + + if (pendingLinks is not { Count: > 0 }) + { + // No pending links, nothing to do. + return; + } + + // Resolve the destination directory against the filesystem root, following any pre-existing symlinks in its path. + string resolvedDestinationDirectoryFullPath; + int destinationSymlinkDepth; + { + string root = Path.GetPathRoot(destinationDirectoryFullPath)!; + var rootResolver = new PendingLinkResolver(root, root, destinationSymlinkDepth: 0, pendingLinks: null); + resolvedDestinationDirectoryFullPath = rootResolver.ResolveDestinationDirectory(destinationDirectoryFullPath.AsSpan(root.Length)); + destinationSymlinkDepth = rootResolver.SymlinkDepth; + + // Ensure DirectorySeparatorChar. + if (!resolvedDestinationDirectoryFullPath.EndsWith(Path.DirectorySeparatorChar)) + { + resolvedDestinationDirectoryFullPath += Path.DirectorySeparatorChar; + } + } + + // Now create all pending links (that have ".." segments) + var resolver = new PendingLinkResolver(destinationDirectoryFullPath, resolvedDestinationDirectoryFullPath, destinationSymlinkDepth, pendingLinks); + + // CreateLink removes the link from pendingLinks once it is created, and it may create (and remove) several + // more when a link resolves through other pending links. Because that mutates the dictionary, we can't + // enumerate it directly, so we take a snapshot of the keys and skip the ones that were already created. + // The order of creation doesn't matter. + FileId[] fileIds = new FileId[pendingLinks.Count]; + pendingLinks.Keys.CopyTo(fileIds, 0); + + foreach (FileId fileId in fileIds) + { + if (pendingLinks.TryGetValue(fileId, out (string SymlinkPath, string LinkTarget) pendingLink)) + { + resolver.CreateLink(fileId, pendingLink.SymlinkPath, pendingLink.LinkTarget); + } + } + + Debug.Assert(pendingLinks.Count == 0); + } + + // checks if a path is equal to or nested under a full path. The full path must end with a directory separator. + // examples: + // FullPathIsEqualOrNested("C:\foo\", "C:\foo\bar") => true + // FullPathIsEqualOrNested("C:\foo\", "C:\foo") => true + // FullPathIsEqualOrNested("C:\foo\", "C:\foobar") => false + internal static bool FullPathIsEqualOrNested(string fullPath, string nestedFullPath) + { + Debug.Assert(fullPath[^1] == Path.DirectorySeparatorChar); + + return nestedFullPath.StartsWith(fullPath, PathInternal.StringComparison) + || string.Equals(nestedFullPath, fullPath.TrimEnd(Path.DirectorySeparatorChar), PathInternal.StringComparison); + } + + // Resolves the targets of deferred (pending) symbolic links produced during extraction, ensuring none of them + // resolve outside the destination directory. Pre-existing symlinks that already point outside the destination + // widen the allowed region: a pending link may point under the location such a symlink resolved to. + private sealed class PendingLinkResolver + { + // Invariant for the lifetime of the resolver. + private readonly string _destination; + private readonly string _resolvedDestination; + private readonly int _destinationSymlinkDepth; + private readonly Dictionary? _pendingLinks; + + // Reset at the start of each top-level link resolution; shared (via ref semantics) across the recursive walk. + private int _symlinkDepth; + + internal PendingLinkResolver(string destination, string resolvedDestination, int destinationSymlinkDepth, Dictionary? pendingLinks) + { + Debug.Assert(destination[^1] == Path.DirectorySeparatorChar); + + _destination = destination; + _resolvedDestination = resolvedDestination; + _destinationSymlinkDepth = destinationSymlinkDepth; + _pendingLinks = pendingLinks; + _symlinkDepth = destinationSymlinkDepth; + } + + internal int SymlinkDepth => _symlinkDepth; + + // Resolves the destination directory itself (used by RealPath). Nothing is ever outside the filesystem root. + internal string ResolveDestinationDirectory(ReadOnlySpan relativePath) + { + string? rootPath = _destination; + Resolve(ref rootPath, _destination, relativePath, out string resolvedPath, visitedLinks: null); + return resolvedPath; + } + + // Resolves and creates a single top-level pending link. + internal void CreateLink(FileId fileId, string symlinkPath, string linkTarget) + { + // Each top-level link starts a fresh resolution walk. + _symlinkDepth = _destinationSymlinkDepth; + AddPendingLink(_resolvedDestination, fileId, symlinkPath, linkTarget, visitedLinks: null, out _); + } + + private void AddPendingLink(string rootPath, FileId fileId, string symlinkPath, string linkTarget, HashSet? visitedLinks, out string resolvedFullPath) + { + Debug.Assert(rootPath is not null); + + string? rootPathLocal = rootPath; + ReadOnlySpan relativeLinkPath = symlinkPath.AsSpan(_destination.Length); + string resolvedPath = _resolvedDestination; + ReadOnlySpan directoryPath = Path.GetDirectoryName(relativeLinkPath); + if (!directoryPath.IsEmpty) + { + if (!Resolve(ref rootPathLocal, resolvedPath, directoryPath, out resolvedPath, visitedLinks)) + { + throw new IOException(SR.Format(SR.TarExtractingResultsLinkOutside, linkTarget, _destination)); + } + Debug.Assert(rootPathLocal is not null); + } + + ReadOnlySpan resolveTarget = linkTarget; + if (Path.IsPathFullyQualified(linkTarget)) + { + rootPathLocal = _resolvedDestination; + resolvedPath = Path.GetPathRoot(linkTarget)!; + resolveTarget = linkTarget.AsSpan(resolvedPath.Length); + } + + if (!Resolve(ref rootPathLocal, resolvedPath, resolveTarget, out resolvedFullPath, visitedLinks)) + { + throw new IOException(SR.Format(SR.TarExtractingResultsLinkOutside, linkTarget, _destination)); + } + Debug.Assert(rootPathLocal is not null); + + if (_pendingLinks!.Remove(fileId)) + { + File.Delete(symlinkPath); + File.CreateSymbolicLink(symlinkPath, linkTarget); + } + } + + // Returns whether the path formed by combining baseFullPath and relativePath resides in either the root path or the destination path. + // The containmentRootPath path tracks a location that may not be in the destination path but which is reachable (via preexisting links) + // and considered a root that a delayed link target may point under. + private bool Resolve(ref string? containmentRootPath, string baseFullPath, ReadOnlySpan relativePath, out string resolvedFullPath, HashSet? visitedLinks) + { + resolvedFullPath = baseFullPath; + bool checkExists = true; + + // walk the relative path and resolve each component, following symlinks and checking for loops. + foreach (Range range in relativePath.SplitAny(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + { + ReadOnlySpan component = relativePath[range]; + if (component.IsEmpty || component is ".") + { + continue; + } + if (component is "..") + { + // Traverse to the parent. Strip any trailing separators first, otherwise + // GetDirectoryName only strips them and doesn't move up a level. + ReadOnlySpan resolvedFullPathSpan = resolvedFullPath.AsSpan(); + while (!resolvedFullPathSpan.IsEmpty && PathInternal.IsDirectorySeparator(resolvedFullPathSpan[^1])) + { + resolvedFullPathSpan = resolvedFullPathSpan[..^1]; + } + + // GetDirectoryName returns null at the root, map back to root (.. on / yields /). + resolvedFullPathSpan = Path.GetDirectoryName(resolvedFullPathSpan); + resolvedFullPath = resolvedFullPathSpan.IsEmpty ? resolvedFullPath : resolvedFullPathSpan.ToString(); + + if (containmentRootPath is not null && !IsInRootOrDestination(containmentRootPath, resolvedFullPath)) + { + return false; + } + continue; + } + + string componentPath = Path.Join(resolvedFullPath, component); + + // TryGetPathInfo throws on error and returns false when the path does not exist. + if (!checkExists || !TryGetPathInfo(componentPath, out FileId fileId, out bool isSymlink)) + { + // Path doesn't exist; remaining components can't exist either. + // short-circuit to simply appending the remaining components to the path for the final check. + resolvedFullPath = componentPath; + checkExists = false; + continue; + } + + string? linkTarget = null; + (string SymlinkPath, string LinkTarget) pendingLink = default; + if (isSymlink) + { + linkTarget = new FileInfo(componentPath).LinkTarget; + } + else if (_pendingLinks?.TryGetValue(fileId, out pendingLink) == true) + { + linkTarget = pendingLink.LinkTarget; + } + + if (linkTarget is null) + { + resolvedFullPath = componentPath; + continue; + } + + if (++_symlinkDepth > MaxSymlinkDepth) + { + throw new IOException(SR.Format(SR.IO_TooManySymbolicLinkLevels, componentPath)); + } + + // To allow creating symbolic links that cause loops we need to detect the loops. + // visitedLinks tracks the links we've followed. + visitedLinks ??= new HashSet(); + if (!visitedLinks.Add(fileId)) + { + // Already visited. + visitedLinks.Clear(); // Clear the set so callers detect circularity via Remove returning false below. + resolvedFullPath = componentPath; + break; + } + + if (isSymlink) + { + if (!FollowExistingSymlink(ref containmentRootPath, resolvedFullPath, linkTarget, visitedLinks, out resolvedFullPath)) + { + Debug.Fail("Should be able to follow existing symlinks"); + return false; + } + } + else + { + // Pending link. A fully qualified target must point inside the destination. + if (Path.IsPathFullyQualified(linkTarget)) + { + containmentRootPath = _resolvedDestination; + } + // If no rootPath was set yet, limit the pending link to the destination dir. + AddPendingLink(containmentRootPath ?? _resolvedDestination, fileId, pendingLink.SymlinkPath!, pendingLink.LinkTarget!, visitedLinks, out resolvedFullPath); + } + + if (!visitedLinks.Remove(fileId)) + { + // Circular reference detected while resolving the link above. + break; + } + } + + return containmentRootPath is null || IsInRootOrDestination(containmentRootPath, resolvedFullPath); + } + + // Follows a pre-existing symlink that is already on disk. Existing symlinks are not limited to a root path; + // after resolving, the root path is widened to the resolved location so pending links may also point under it. + private bool FollowExistingSymlink(ref string? rootPath, string linkDirectory, string linkTarget, HashSet? visitedLinks, out string resolvedFullPath) + { + string resolveFrom; + ReadOnlySpan resolveTarget; + if (Path.IsPathFullyQualified(linkTarget)) + { + resolveFrom = Path.GetPathRoot(linkTarget)!; + resolveTarget = linkTarget.AsSpan(resolveFrom.Length); + } + else + { + resolveFrom = linkDirectory; + resolveTarget = linkTarget; + } + + // set root path as null to disable the containment checks inside Resolve + string? symlinkRootPath = null; + if (!Resolve(ref symlinkRootPath, resolveFrom, resolveTarget, out resolvedFullPath, visitedLinks)) + { + return false; + } + + rootPath = resolvedFullPath; + if (!rootPath.EndsWith(Path.DirectorySeparatorChar)) + { + rootPath += Path.DirectorySeparatorChar; + } + return true; + } + + private bool IsInRootOrDestination(string rootPath, string resolvedPath) + => FullPathIsEqualOrNested(rootPath, resolvedPath) + || (!ReferenceEquals(rootPath, _resolvedDestination) && FullPathIsEqualOrNested(_resolvedDestination, resolvedPath)); + } } } diff --git a/src/libraries/System.Formats.Tar/tests/TarFile/TarFile.ExtractToDirectory.File.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarFile/TarFile.ExtractToDirectory.File.Tests.cs index 0ac04584ce96a3..83e1910aeae563 100644 --- a/src/libraries/System.Formats.Tar/tests/TarFile/TarFile.ExtractToDirectory.File.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarFile/TarFile.ExtractToDirectory.File.Tests.cs @@ -10,6 +10,19 @@ namespace System.Formats.Tar.Tests { public partial class TarFile_ExtractToDirectory_File_Tests : TarTestsBase { + // ensures that the link name is in the correct format for the current OS. + private static string NormalizePathSeparators(string path) + { + if (OperatingSystem.IsWindows()) + { + return path.Replace('/', '\\'); + } + else + { + return path; + } + } + [Fact] public void InvalidPaths_Throw() { @@ -352,7 +365,7 @@ public void LinkBeforeTarget() using (TarWriter writer = new TarWriter(archiveStream)) { PaxTarEntry link = new PaxTarEntry(TarEntryType.SymbolicLink, "link"); - link.LinkName = "dir/file"; + link.LinkName = NormalizePathSeparators("dir/file"); writer.WriteEntry(link); PaxTarEntry file = new PaxTarEntry(TarEntryType.RegularFile, "dir/file"); @@ -387,7 +400,7 @@ public void ExtractToDirectory_RejectsSymlinkDirectoryTraversal_WithNestedFile() // symlink: "link" -> "/tmp/outside" writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") { - LinkName = linkTarget + LinkName = NormalizePathSeparators(linkTarget) }); // file: "link/test.txt" with "hello" @@ -434,7 +447,7 @@ public void ExtractToDirectory_RejectsChainedSymlinkDirectoryTraversal_WithNeste writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b/c") { LinkName = "." }); - writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b/c/d") { LinkName = "../../outside" }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b/c/d") { LinkName = NormalizePathSeparators("../../outside") }); var pwned = new PaxTarEntry(TarEntryType.RegularFile, "a/d/pwned.txt") { @@ -467,5 +480,452 @@ public void ExtractToDirectory_RejectsChainedSymlinkDirectoryTraversal_WithNeste Assert.False(File.Exists(Path.Combine(outsideDir, "pwned.txt")), "pwned.txt should not have been written outside destination."); } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_RejectsSymlinkDirectoryTraversal() + { + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "symlink_dir_traversal.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + // symlink: "link" -> "/tmp/outside" (absolute path outside destDir) + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") + { + LinkName = NormalizePathSeparators("/tmp/outside") + }); + } + + Assert.Throws(() => TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true)); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DefersSymlinkWithDotDot() + { + // Symlinks whose target contains '..' are deferred: a placeholder file is created + // during extraction, and the real symlink replaces it after all entries are processed. + // This prevents subsequent entries from following the symlink to escape the destination. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "deferred_symlink.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "a", "link"); + FileSystemInfo linkInfo = new FileInfo(linkPath); + Assert.Equal(NormalizePathSeparators("../b"), linkInfo.LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DefersSymlinkWithDotDot_UnderSymlinkedParent() + { + using TempDirectory root = new TempDirectory(); + string realRoot = Path.Combine(root.Path, "real"); + Directory.CreateDirectory(realRoot); + + string linkedRoot = Path.Combine(root.Path, "link"); + Directory.CreateSymbolicLink(linkedRoot, realRoot); + + string destDir = Path.Combine(linkedRoot, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "deferred_symlink.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "a", "link"); + Assert.Equal(NormalizePathSeparators("../b"), new FileInfo(linkPath).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DeferredSymlinkBlocksTraversal() + { + // A file entry after a deferred symlink cannot follow the symlink to write + // outside the destination directory, because the symlink is a placeholder file + // during extraction. + // + // dir a/ + // symlink a/b -> . + // symlink a/b/c -> . + // symlink a/b/c/escape -> ../../.. (deferred: contains '..') + // file a/b/c/escape/pwned (would escape via chained symlinks if not deferred) + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "chained_symlink_traversal.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b") { LinkName = "." }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b/c") { LinkName = "." }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/b/c/escape") { LinkName = NormalizePathSeparators("../../..") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.RegularFile, "a/b/c/escape/pwned")); + } + + // The file entry "a/b/c/escape/pwned" cannot follow the 'escape' symlink because + // it is a placeholder file during extraction. Creating the file's parent directory + // will fail because 'escape' is a regular file, not a directory. + Assert.ThrowsAny(() => TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true)); + + // The file must not exist at the root (where the symlink chain would have led). + Assert.False(File.Exists(Path.Combine(root.Path, "pwned"))); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_AllowsSymlinkToDestinationDirectory() + { + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "symlink_to_dest.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + // symlink: "link" -> "." (points to the destination directory itself) + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") { LinkName = "." }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "link"); + FileSystemInfo linkInfo = new FileInfo(linkPath); + Assert.Equal(".", linkInfo.LinkTarget); + } + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DeferredSymlinkAllowedUnderExistingExternalSymlink() + { + // When the destination contains an existing symlink that points outside, + // a deferred symlink (with '..') under that external location is allowed + // as long as it stays within the directory the existing symlink resolved to. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + // Create an external directory and an existing symlink in dest pointing to it. + string externalDir = Path.Combine(root.Path, "external"); + Directory.CreateDirectory(Path.Combine(externalDir, "sub")); + Directory.CreateSymbolicLink(Path.Combine(destDir, "ext"), externalDir); + + string tarPath = Path.Combine(root.Path, "deferred_under_external.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + // ext/sub/link -> .. (resolves to externalDir, stays within it) + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "ext/sub/link") { LinkName = ".." }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "ext", "sub", "link"); + Assert.Equal("..", new FileInfo(linkPath).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DeferredSymlinkRejectedWhenEscapingExternalSymlink() + { + // When the destination contains an existing symlink that points outside, + // a deferred symlink that escapes beyond that external directory is rejected. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string externalDir = Path.Combine(root.Path, "external"); + Directory.CreateDirectory(externalDir); + Directory.CreateSymbolicLink(Path.Combine(destDir, "ext"), externalDir); + + string tarPath = Path.Combine(root.Path, "escape_external.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + // ext/link -> ../.. (escapes beyond externalDir) + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "ext/link") { LinkName = NormalizePathSeparators("../..") }); + } + + Assert.Throws(() => TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true)); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_CircularDeferredSymlinksAllowed() + { + // Circular symlinks that contain '..' but stay within the destination are allowed. + // The cycle detection (visitedLinks) prevents infinite recursion during resolution. + // + // a/link1 -> ../b + // b/link2 -> ../a + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "circular_deferred.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link1") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "b/link2") { LinkName = NormalizePathSeparators("../a") }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string link1Path = Path.Combine(destDir, "a", "link1"); + string link2Path = Path.Combine(destDir, "b", "link2"); + Assert.Equal(NormalizePathSeparators("../b"), new FileInfo(link1Path).LinkTarget); + Assert.Equal(NormalizePathSeparators("../a"), new FileInfo(link2Path).LinkTarget); + } + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_OverwriteDeferredSymlink() + { + // When a second symlink entry overwrites a previously deferred symlink, + // the placeholder is replaced and the final symlink uses the second target. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "overwrite_deferred.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "c/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators("../c") }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "a", "link"); + Assert.Equal(NormalizePathSeparators("../c"), new FileInfo(linkPath).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_RegularFileOverwritesDeferredSymlink() + { + // When a regular file entry overwrites a previously deferred symlink, + // the placeholder is replaced by the file and its pending link entry is + // dropped, so the final entry is the file (not the symlink). + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "file_overwrites_deferred.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.RegularFile, "a/link") + { + DataStream = new MemoryStream(Encoding.UTF8.GetBytes("content")) + }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "a", "link"); + Assert.Null(new FileInfo(linkPath).LinkTarget); + Assert.Equal("content", File.ReadAllText(linkPath)); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_AbsoluteSymlinkWithinDestination() + { + // A symlink with a fully qualified target that points within the destination + // is allowed and is not deferred (no '..'). + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string absoluteTarget = Path.Combine(destDir, "target"); + + string tarPath = Path.Combine(root.Path, "absolute_symlink.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "target/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") { LinkName = NormalizePathSeparators(absoluteTarget) }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "link"); + Assert.Equal(absoluteTarget, new FileInfo(linkPath).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_DeferredSymlinkWithAbsoluteTarget() + { + // A deferred symlink (contains '..') with a fully qualified target that + // resolves within the destination is allowed. Exercises the + // rootPath = resolvedDestinationPath path in ResolvePathInRootOrDestination. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + // Absolute target with '..' that resolves back into dest. + string absoluteTarget = NormalizePathSeparators(Path.Combine(destDir, "a", "..", "b")); + + string tarPath = Path.Combine(root.Path, "deferred_absolute.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") { LinkName = absoluteTarget }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string linkPath = Path.Combine(destDir, "link"); + // Windows APIs collapse the intermediate ".." in the link target, so the expected target is destDir/b. On Unix, the link target is preserved as-is. + string expectedTarget = OperatingSystem.IsWindows() ? Path.Combine(destDir, "b") : absoluteTarget; + Assert.Equal(expectedTarget, new FileInfo(linkPath).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_ChainedDeferredSymlinks() + { + // Two deferred symlinks where resolving the second one's target + // passes through the first one, forcing recursive resolution + // during CreatePendingLinks. + // + // a/link1 -> ../b (deferred: contains '..') + // a/link2 -> link1/../b/sub (deferred: contains '..', + // resolving encounters link1 as pending link, + // triggering recursive creation of link1 first) + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "chained_deferred.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/sub/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link1") { LinkName = NormalizePathSeparators("../b") }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link2") { LinkName = NormalizePathSeparators("link1/../b/sub") }); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + string link1Path = Path.Combine(destDir, "a", "link1"); + string link2Path = Path.Combine(destDir, "a", "link2"); + Assert.Equal(NormalizePathSeparators("../b"), new FileInfo(link1Path).LinkTarget); + Assert.Equal(NormalizePathSeparators("link1/../b/sub"), new FileInfo(link2Path).LinkTarget); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_EscapeViaSymlinkToCurrentDirectory_Throws() + { + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "escape.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a") { LinkName = "." }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "link") { LinkName = NormalizePathSeparators("a/../outside") }); + } + + Assert.Throws(() => TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true)); + } + + [ConditionalFact(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + public void ExtractToDirectory_ExceedsMaxSymlinkDepth() + { + // A chain of deferred symlinks that exceeds MaxSymlinkDepth (40) + // should throw during CreatePendingLinks. + // + // Each link_N -> ../link_{N-1} (deferred because of '..'). + // Resolving link_41's target chains through all 41 preceding + // symlinks, exceeding the depth limit. + + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(destDir); + + string tarPath = Path.Combine(root.Path, "deep_symlinks.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + const int count = 42; + for (int i = 0; i < count; i++) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, $"d{i}/")); + } + // link_0 -> ../d0 (base case) + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "d0/link_0") { LinkName = NormalizePathSeparators("../d0") }); + // link_N -> ../d{N-1}/link_{N-1} — each follows the previous, all deferred + for (int i = 1; i < count; i++) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, $"d{i}/link_{i}") { LinkName = NormalizePathSeparators($"../d{i - 1}/link_{i - 1}") }); + } + } + + Assert.Throws(() => TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true)); + } + + [ConditionalTheory(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks))] + [InlineData("../b")] // deferred: contains '..' + [InlineData("b")] // not deferred + public void ExtractToDirectory_OverwritesExistingDanglingSymlink(string linkName) + { + using TempDirectory root = new TempDirectory(); + string destDir = Path.Combine(root.Path, "dest"); + Directory.CreateDirectory(Path.Combine(destDir, "a")); + + // A pre-existing symlink whose target does not exist. + File.CreateSymbolicLink(Path.Combine(destDir, "a", "link"), "does_not_exist"); + + string tarPath = Path.Combine(root.Path, "overwrite_dangling.tar"); + using (FileStream stream = new FileStream(tarPath, FileMode.Create, FileAccess.Write)) + using (TarWriter writer = new TarWriter(stream, leaveOpen: false)) + { + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "a/")); + writer.WriteEntry(new PaxTarEntry(TarEntryType.SymbolicLink, "a/link") { LinkName = NormalizePathSeparators(linkName) }); + writer.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "b/")); + } + + TarFile.ExtractToDirectory(tarPath, destDir, overwriteFiles: true); + + Assert.Equal(NormalizePathSeparators(linkName), new FileInfo(Path.Combine(destDir, "a", "link")).LinkTarget); + } } } diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs index a1256da552f63f..91537c3b9c39eb 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/RedirectHandler.cs @@ -50,8 +50,9 @@ internal override async ValueTask SendAsync(HttpRequestMess response.Dispose(); - // Clear the authorization header. + // Clear the authorization headers. request.Headers.Authorization = null; + request.Headers.ProxyAuthorization = null; if (HttpTelemetry.Log.IsEnabled()) { diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/DateTimeParse.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/DateTimeParse.cs index bec15897701aa1..cdc4e30527acce 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/DateTimeParse.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/DateTimeParse.cs @@ -624,7 +624,7 @@ private static bool Lex(DS dps, ref __DTString str, scoped ref DateTimeToken dto { case TokenType.NumberToken: case TokenType.YearNumberToken: - if (raw.numCount == 3 || tokenValue == -1) + if (raw.numCount == DateTimeRawInfo.MaxBufferLength || tokenValue == -1) { result.SetBadDateTimeFailure(); LexTraceExit("0010", dps); @@ -921,6 +921,13 @@ private static bool Lex(DS dps, ref __DTString str, scoped ref DateTimeToken dto else { // This is a day number + if (raw.numCount == DateTimeRawInfo.MaxBufferLength) + { + result.SetBadDateTimeFailure(); + LexTraceExit("0105 (Not enough space in the raw info buffer when parsing Hebrew numbers)", dps); + return false; + } + dtok.num = tokenValue; raw.AddNumber(dtok.num); @@ -2528,7 +2535,7 @@ internal static bool TryParse(ReadOnlySpan s, DateTimeFormatInfo dtfi, Dat DateTimeRawInfo raw = default; // The buffer to store temporary parsing information. unsafe { - int* numberPointer = stackalloc int[3]; + int* numberPointer = stackalloc int[DateTimeRawInfo.MaxBufferLength]; raw.Init(numberPointer); } raw.hasSameDateAndTimeSeparators = dtfi.DateSeparator.Equals(dtfi.TimeSeparator, StringComparison.Ordinal); @@ -5983,6 +5990,7 @@ struct DateTimeToken // internal unsafe struct DateTimeRawInfo { + internal const int MaxBufferLength = 3; private int* num; internal int numCount; internal int month; @@ -6006,7 +6014,16 @@ internal void Init(int* numberBuffer) internal void AddNumber(int value) { + if (numCount >= MaxBufferLength) + { + // defense in depth. we shouldn't come here though. + ThrowNumberBufferOverflow(); + } + num[numCount++] = value; + + [DoesNotReturn] + static void ThrowNumberBufferOverflow() => throw new InvalidOperationException(SR.InvalidOperation_DateTimeParsing); } internal int GetNumber(int index) diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Core/LocalAppContextSwitches.cs b/src/libraries/System.Private.Xml/src/System/Xml/Core/LocalAppContextSwitches.cs index a95f9105051da1..1bd544cd7e6c71 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Core/LocalAppContextSwitches.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Core/LocalAppContextSwitches.cs @@ -67,5 +67,16 @@ public static bool IsNetworkingEnabledByDefault return SwitchesHelpers.GetCachedSwitchValue("System.Xml.XmlResolver.IsNetworkingEnabledByDefault", ref s_isNetworkingEnabledByDefault); } } + private static int s_disableOptimizedXmlTextReaderAttributeBufferInvalidation; + public static bool DisableOptimizedXmlTextReaderAttributeBufferInvalidation + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + return SwitchesHelpers.GetCachedSwitchValue( + "Switch.System.Xml.DisableOptimizedXmlTextReaderAttributeBufferInvalidation", + ref s_disableOptimizedXmlTextReaderAttributeBufferInvalidation); + } + } } } diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImpl.cs b/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImpl.cs index 236f166925f5a7..af76d45ae8df06 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImpl.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImpl.cs @@ -163,6 +163,7 @@ private enum InitInputType // attributes info private int _curAttrIndex = -1; private int _attrCount; + private int _attrBufferInvalidationIndex; private int _attrHashtable; private int _attrDuplWalkCount; private bool _attrNeedNamespaceLookup; @@ -3345,10 +3346,25 @@ private int ReadData() // the character buffer is full -> allocate a new one if (_ps.charsUsed == _ps.chars.Length - 1) { - // invalidate node values kept in buffer - applies to attribute values only - for (int i = 0; i < _attrCount; i++) + if (LocalAppContextSwitches.DisableOptimizedXmlTextReaderAttributeBufferInvalidation) { - _nodes[_index + i + 1].OnBufferInvalidated(); + for (int i = 0; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + } + else + { + // invalidate node values kept in buffer - applies to attribute values only. + // Attribute values below _attrBufferInvalidationIndex have already been materialized + // into strings and no longer reference the buffer. The last attribute may still be in + // the middle of being re-parsed and can have its buffer-backed value assigned after this + // point, so it is intentionally left to be re-checked on the next buffer reload. + for (int i = _attrBufferInvalidationIndex; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + _attrBufferInvalidationIndex = Math.Max(_attrCount - 1, 0); } char[] newChars = new char[_ps.chars.Length * 2]; @@ -3382,10 +3398,25 @@ private int ReadData() int charsLen = _ps.chars.Length; if (charsLen - _ps.charsUsed <= charsLen / 2) { - // invalidate node values kept in buffer - applies to attribute values only - for (int i = 0; i < _attrCount; i++) + if (LocalAppContextSwitches.DisableOptimizedXmlTextReaderAttributeBufferInvalidation) { - _nodes[_index + i + 1].OnBufferInvalidated(); + for (int i = 0; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + } + else + { + // invalidate node values kept in buffer - applies to attribute values only. + // Attribute values below _attrBufferInvalidationIndex have already been materialized + // into strings and no longer reference the buffer. The last attribute may still be in + // the middle of being re-parsed and can have its buffer-backed value assigned after this + // point, so it is intentionally left to be re-checked on the next buffer reload. + for (int i = _attrBufferInvalidationIndex; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + _attrBufferInvalidationIndex = Math.Max(_attrCount - 1, 0); } // move unparsed characters to front, unless the whole buffer contains unparsed characters @@ -7817,6 +7848,7 @@ private void ResetAttributes() _curAttrIndex = -1; _attrCount = 0; + _attrBufferInvalidationIndex = 0; _attrHashtable = 0; _attrDuplWalkCount = 0; } diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImplAsync.cs b/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImplAsync.cs index 8edafd8e7ef408..dcda73aa9f91b2 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImplAsync.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Core/XmlTextReaderImplAsync.cs @@ -1034,10 +1034,25 @@ private async Task ReadDataAsync() // the character buffer is full -> allocate a new one if (_ps.charsUsed == _ps.chars.Length - 1) { - // invalidate node values kept in buffer - applies to attribute values only - for (int i = 0; i < _attrCount; i++) + if (LocalAppContextSwitches.DisableOptimizedXmlTextReaderAttributeBufferInvalidation) { - _nodes[_index + i + 1].OnBufferInvalidated(); + for (int i = 0; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + } + else + { + // invalidate node values kept in buffer - applies to attribute values only. + // Attribute values below _attrBufferInvalidationIndex have already been materialized + // into strings and no longer reference the buffer. The last attribute may still be in + // the middle of being re-parsed and can have its buffer-backed value assigned after this + // point, so it is intentionally left to be re-checked on the next buffer reload. + for (int i = _attrBufferInvalidationIndex; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + _attrBufferInvalidationIndex = Math.Max(_attrCount - 1, 0); } char[] newChars = new char[_ps.chars.Length * 2]; @@ -1070,10 +1085,25 @@ private async Task ReadDataAsync() int charsLen = _ps.chars.Length; if (charsLen - _ps.charsUsed <= charsLen / 2) { - // invalidate node values kept in buffer - applies to attribute values only - for (int i = 0; i < _attrCount; i++) + if (LocalAppContextSwitches.DisableOptimizedXmlTextReaderAttributeBufferInvalidation) { - _nodes[_index + i + 1].OnBufferInvalidated(); + for (int i = 0; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + } + else + { + // invalidate node values kept in buffer - applies to attribute values only. + // Attribute values below _attrBufferInvalidationIndex have already been materialized + // into strings and no longer reference the buffer. The last attribute may still be in + // the middle of being parsed and can have its buffer-backed value assigned after this + // point, so it is intentionally left to be re-checked on the next buffer reload. + for (int i = _attrBufferInvalidationIndex; i < _attrCount; i++) + { + _nodes[_index + i + 1].OnBufferInvalidated(); + } + _attrBufferInvalidationIndex = Math.Max(_attrCount - 1, 0); } // move unparsed characters to front, unless the whole buffer contains unparsed characters diff --git a/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj b/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj index f4a55b35853065..9b942d256fad1e 100644 --- a/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj @@ -328,6 +328,7 @@ + diff --git a/src/libraries/System.Private.Xml/tests/XmlReader/Tests/AttributeBufferInvalidationTests.cs b/src/libraries/System.Private.Xml/tests/XmlReader/Tests/AttributeBufferInvalidationTests.cs new file mode 100644 index 00000000000000..29b489d4fa2ad4 --- /dev/null +++ b/src/libraries/System.Private.Xml/tests/XmlReader/Tests/AttributeBufferInvalidationTests.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +namespace System.Xml.XmlReaderTests +{ + public static class AttributeBufferInvalidationTests + { + // The reader keeps attribute values as lazy references into its internal character buffer and only + // materializes ("invalidates") them into strings when that buffer has to be grown or shifted while a + // start tag is still being parsed. Parsing an element whose start tag is much larger than the internal + // buffer (4 KB for the synchronous reader, 32 KB for the asynchronous one) forces ReadData/ReadDataAsync + // to reload the buffer many times, so the invalidation loop runs multiple passes over the attribute nodes. + // + // These tests exercise that path and verify that every attribute value survives the reloads intact. + // Multiple elements ensure the per-element invalidation bookkeeping is reset between elements, and one + // attribute per element is deliberately written with a huge run of whitespace between its name and the + // '=' sign: because that run is larger than the internal buffer, a buffer reload is guaranteed to happen + // after the attribute has been counted but before its buffer-backed value has been assigned. That is + // exactly the window in which the bookkeeping must not mark the still-pending attribute as already + // materialized; a regression there leaves a dangling reference into the buffer and corrupts the value. + private const int ElementCount = 3; + private const int AttributesPerElement = 3000; + private const int ValuePadding = 40; + + // Larger than the asynchronous reader's 32 KB internal buffer so a reload is guaranteed while the + // attribute carrying it is counted but not yet valued. + private const int PendingAttributeWhitespace = 40 * 1024; + + // The attribute (per element) that is preceded by the huge whitespace run. + private const int PendingAttributeIndex = 1; + + private static string MakeAttributeValue(int element, int attribute) => + $"E{element}_A{attribute}_" + new string((char)('a' + (attribute % 26)), ValuePadding); + + private static string BuildXml() + { + var sb = new StringBuilder(ElementCount * (PendingAttributeWhitespace + AttributesPerElement * (ValuePadding + 24))); + sb.Append(""); + for (int e = 0; e < ElementCount; e++) + { + sb.Append(""); + } + sb.Append(""); + return sb.ToString(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public static async Task AttributeValuesRemainValidAcrossBufferReloads(bool async) + { + string xml = BuildXml(); + var settings = new XmlReaderSettings { Async = async }; + + using XmlReader reader = XmlReader.Create(new StringReader(xml), settings); + + int elementsChecked = 0; + while (async ? await reader.ReadAsync() : reader.Read()) + { + if (reader.NodeType != XmlNodeType.Element || reader.Depth == 0) + { + continue; + } + + Assert.Equal($"e{elementsChecked}", reader.Name); + Assert.Equal(AttributesPerElement, reader.AttributeCount); + + for (int a = 0; a < AttributesPerElement; a++) + { + Assert.Equal(MakeAttributeValue(elementsChecked, a), reader.GetAttribute(a)); + } + + elementsChecked++; + } + + Assert.Equal(ElementCount, elementsChecked); + } + } +} diff --git a/src/libraries/System.Runtime.Serialization.Xml/tests/Canonicalization/XmlCanonicalizationTest.cs b/src/libraries/System.Runtime.Serialization.Xml/tests/Canonicalization/XmlCanonicalizationTest.cs index d8db0ea42ff757..4b9bcba6d7c8e0 100644 --- a/src/libraries/System.Runtime.Serialization.Xml/tests/Canonicalization/XmlCanonicalizationTest.cs +++ b/src/libraries/System.Runtime.Serialization.Xml/tests/Canonicalization/XmlCanonicalizationTest.cs @@ -300,7 +300,8 @@ public static void ReaderWriter_C14N_DifferentReadersWriters() } //TestC14NWriterWithManyAttributes - int numberOfAttributes = 1000; + // Stay below the default per-element C14N limit while preserving the interoperability stress test. + const int numberOfAttributes = 500; int seed = (int)DateTime.Now.Ticks; Random rndGen = new Random(seed); StringBuilder sb = new StringBuilder(); diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DateTimeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DateTimeTests.cs index 554d128a15933d..d6e3be6ded7c47 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DateTimeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DateTimeTests.cs @@ -1966,6 +1966,30 @@ public static void TestTryParseAtBoundaries() "DateTime parsing expected to throw with any dates greater than DateTime.MaxValue"); } + private static bool HebrewNumberLexingSupported => + PlatformDetection.IsNotInvariantGlobalization && PlatformDetection.IsNotHybridGlobalizationOnApplePlatform; + + [ConditionalTheory(typeof(DateTimeTests), nameof(HebrewNumberLexingSupported))] + [InlineData("\u6642")] // CJK hour suffix 時 + [InlineData("\u5206")] // CJK minute suffix 分 + [InlineData("\u79D2")] // CJK second suffix 秒 + public static void TryParse_TooManyNumbersViaHebrewLexer_FailsWithoutOverflow(string timeSuffix) + { + // Any letter in the Hebrew Unicode block (U+0590-U+05FF) is lexed as a "Hebrew number" + // under every non-invariant culture -- this does NOT require a Hebrew calendar or a Hebrew + // culture. The CJK hour/minute/second suffixes are likewise recognized everywhere. Using + // spaces as separators (a separator in every culture) makes the repro culture-independent: + // "1" and "2" fill two of DateTimeRawInfo's three number slots; + // "3" is consumed as a time-suffixed number without taking a slot; + // the Hebrew numerals are then lexed as additional day numbers, and the second one + // attempts a fourth push into the three-slot buffer. + // Previously that fourth push overflowed the buffer and corrupted adjacent memory; ensure + // the input now fails to parse cleanly instead. + // Example (hour suffix): "1 2 3時 א' ב' ג'" (\u05D0='א', \u05D1='ב', \u05D2='ג', \u0027=geresh) + string input = $"1 2 3{timeSuffix} \u05D0\u0027 \u05D1\u0027 \u05D2\u0027"; + Assert.False(DateTime.TryParse(input, CultureInfo.InvariantCulture, DateTimeStyles.None, out _), $"Input '{input}' should fail to parse."); + } + public static IEnumerable Parse_ValidInput_Succeeds_MemberData() { yield return new object[] { "1234 12", CultureInfo.InvariantCulture, new DateTime(1234, 12, 1, 0, 0, 0) }; diff --git a/src/libraries/System.Security.Cryptography.Pkcs/src/Internal/Cryptography/Pal/Windows/HelpersWindows.cs b/src/libraries/System.Security.Cryptography.Pkcs/src/Internal/Cryptography/Pal/Windows/HelpersWindows.cs index 2ea444ed23690d..3a3b3bfe9703c2 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/src/Internal/Cryptography/Pal/Windows/HelpersWindows.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/src/Internal/Cryptography/Pal/Windows/HelpersWindows.cs @@ -496,7 +496,7 @@ private static unsafe void AddCryptAttribute(CryptographicAttributeObjectCollect attributeCollection.Add(attributeObject); } - collection.Add(new CryptographicAttributeObject(oid, attributeCollection)); + collection.AddWithoutMerge(new CryptographicAttributeObject(oid, attributeCollection)); } } } diff --git a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/Rfc3161TimestampRequest.cs b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/Rfc3161TimestampRequest.cs index 23df99670fcbbf..64482aba5a01a5 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/Rfc3161TimestampRequest.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/Rfc3161TimestampRequest.cs @@ -464,7 +464,7 @@ private Rfc3161RequestResponseStatus ValidateResponse( // If no certificate was requested then the CMS Certificates collection // MUST be empty. - if (tokenCms.Certificates.Count != 0) + if (tokenCms.GetCertificatesNoClone().Count != 0) { if (shouldThrow) { diff --git a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignedCms.cs b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignedCms.cs index c959deede696c3..e55a7877f248b1 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignedCms.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignedCms.cs @@ -19,6 +19,7 @@ public sealed partial class SignedCms { private SignedDataAsn _signedData; private bool _hasData; + private readonly CertificateList _certificates; private readonly SubjectIdentifierType _signerIdentifierType; // A defensive copy of the relevant portions of the data to Decode @@ -71,6 +72,17 @@ public SignedCms(SubjectIdentifierType signerIdentifierType, ContentInfo content ContentInfo = contentInfo; Detached = detached; Version = 0; + _certificates = new CertificateList(this); + } + + internal X509Certificate2? FindCertificate(SubjectIdentifier signerIdentifier) + { + return _certificates.FindCertificate(signerIdentifier); + } + + internal X509Certificate2Collection GetCertificatesNoClone() + { + return _certificates.GetCollection(); } public X509Certificate2Collection Certificates @@ -84,19 +96,16 @@ public X509Certificate2Collection Certificates return coll; } - CertificateChoiceAsn[]? certChoices = _signedData.CertificateSet; - - if (certChoices == null) + foreach (X509Certificate2 cert in GetCertificatesNoClone()) { - return coll; - } - - foreach (CertificateChoiceAsn choice in certChoices) - { - if (choice.Certificate.HasValue) - { - coll.Add(X509CertificateLoader.LoadCertificate(choice.Certificate.Value.Span)); - } + coll.Add( + X509CertificateLoader.LoadCertificate( +#if NET + cert.RawDataMemory.Span +#else + cert.RawData +#endif + )); } return coll; @@ -568,7 +577,7 @@ internal void UpdateCertificatesFromAddition(X509Certificate2Collection newCerts if (existingLength > 0 || newCerts.Count > 1) { - var certs = new HashSet(Certificates.OfType()); + var certs = new HashSet(GetCertificatesNoClone().OfType()); for (int i = 0; i < newCerts.Count; i++) { @@ -728,5 +737,156 @@ public void RemoveCertificate(X509Certificate2 certificate) throw new CryptographicException(SR.Cryptography_Cms_NoCertificateFound); } + + private sealed class CertificateList + { + private readonly SignedCms _parent; + private X509Certificate2Collection? _coll; + private CertificateChoiceAsn[]? _arr; + private Dictionary? _skiLookup; + private Dictionary? _issuerSerialLookup; + + internal CertificateList(SignedCms parent) + { + _parent = parent; + } + + internal X509Certificate2Collection GetCollection() + { + CertificateChoiceAsn[]? certSet = _parent._signedData.CertificateSet; + + if (!ReferenceEquals(_arr, certSet)) + { + _coll = null; + _skiLookup = null; + _issuerSerialLookup = null; + } + + if (_coll == null) + { + X509Certificate2Collection newColl = new(); + + if (certSet != null) + { + foreach (CertificateChoiceAsn cert in certSet) + { + if (cert.Certificate is not null) + { + newColl.Add(X509CertificateLoader.LoadCertificate(cert.Certificate.Value.Span)); + } + } + } + + _coll = newColl; + _arr = certSet; + } + + return _coll; + } + + internal X509Certificate2? FindCertificate(SubjectIdentifier signerIdentifier) + { + const int LinearSearchThreshold = 10; + + X509Certificate2Collection coll = GetCollection(); + + if (coll.Count > LinearSearchThreshold) + { + if (signerIdentifier.Type == SubjectIdentifierType.SubjectKeyIdentifier) + { + if (_skiLookup is null) + { + _skiLookup = new Dictionary(coll.Count, StringComparer.Ordinal); + + foreach (X509Certificate2 item in coll) + { + string ski = PkcsPal.Instance.GetSubjectKeyIdentifier(item).ToSkiString(); + + // The first certificate in the collection with the same SKI wins, + // since the linear method returns the first match. +#if NET + _ = _skiLookup.TryAdd(ski, item); +#else + try + { + _skiLookup.Add(ski, item); + } + catch (ArgumentException) + { + } +#endif + } + } + + string key = (string)signerIdentifier.Value!; + key = key.ToUpperInvariant(); + + Debug.Assert(signerIdentifier.Value is not null); + _ = _skiLookup.TryGetValue(key, out X509Certificate2? cert); + return CloneCert(cert); + } + else if (signerIdentifier.Type == SubjectIdentifierType.IssuerAndSerialNumber) + { + if (_issuerSerialLookup is null) + { + _issuerSerialLookup = new Dictionary(coll.Count, StringComparer.Ordinal); + + foreach (X509Certificate2 item in coll) + { + // Serial number is a hexadecimal string, so it cannot contain a pipe character. + // Therefore, the separator cannot be caller-influenced. + string itemKey = $"{item.SerialNumber}||{item.IssuerName.Name}"; + + // The first certificate in the collection with the same issuer/serial wins. +#if NET + _ = _issuerSerialLookup.TryAdd(itemKey, item); +#else + try + { + _issuerSerialLookup.Add(itemKey, item); + } + catch (ArgumentException) + { + } +#endif + } + } + + var issuerSerial = (Xml.X509IssuerSerial)signerIdentifier.Value!; + string key = $"{issuerSerial.SerialNumber.ToUpperInvariant()}||{issuerSerial.IssuerName}"; + + _ = _issuerSerialLookup.TryGetValue(key, out X509Certificate2? cert); + return CloneCert(cert); + } + else if (signerIdentifier.Type == SubjectIdentifierType.NoSignature) + { + return null; + } + else + { + // Fail in debug, but at runtime fall back to the other function. + Debug.Fail($"Missing handler for SubjectIdentifierType {signerIdentifier.Type}"); + } + } + + return SignerInfo.FindSignerCertificate(signerIdentifier, coll); + + static X509Certificate2? CloneCert(X509Certificate2? cert) + { + if (cert is null) + { + return null; + } + + return X509CertificateLoader.LoadCertificate( +#if NET + cert.RawDataMemory.Span +#else + cert.RawData +#endif + ); + } + } + } } } diff --git a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignerInfo.cs b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignerInfo.cs index 46b6fc056e4c1b..d8e27010712bd4 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignerInfo.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/src/System/Security/Cryptography/Pkcs/SignerInfo.cs @@ -486,10 +486,10 @@ private bool CheckHash(bool compatMode) private X509Certificate2? FindSignerCertificate() { - return FindSignerCertificate(SignerIdentifier, _document.Certificates); + return _document.FindCertificate(SignerIdentifier); } - private static X509Certificate2? FindSignerCertificate( + internal static X509Certificate2? FindSignerCertificate( SubjectIdentifier signerIdentifier, X509Certificate2Collection? extraStore) { diff --git a/src/libraries/System.Security.Cryptography.Pkcs/tests/EnvelopedCms/UnprotectedAttributeTests.cs b/src/libraries/System.Security.Cryptography.Pkcs/tests/EnvelopedCms/UnprotectedAttributeTests.cs index a558ac405b5faf..90e970a076f7bb 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/tests/EnvelopedCms/UnprotectedAttributeTests.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/tests/EnvelopedCms/UnprotectedAttributeTests.cs @@ -471,6 +471,41 @@ public static void TestVersionNumber_RoundTrip(bool addUnprotectedAttrs, int exp Assert.Equal(expectedVersion, ecms.Version); } + [Fact] + public static void DecodeDoesNotMergeAttributes() + { + // This payload has two unprotected attribute sets, both the exact same signing time. + // If the implementation merges them via CryptographicAttributeObjectCollection.Add, + // then we'd notice two things: + // 1) cms.UnprotectedAttributes.Count would be 1 instead of 2 + // 2) The document won't load, since CAOC.Add blocks multiple signing times. + // + // Both elements get tested here to make sure we don't just add a merge allowing duplicate signing times. + byte[] encoded = ( + "308201D706092A864886F70D010703A08201C8308201C4020102318201413082013D0201003025301" + + "8311630140603550403130D706F632D726563697069656E74020900E67536515F29D64B300D06092A" + + "864886F70D010101050004820100B8CC481FF8BED28B411370842708D4551C2422E3F8A0C7E1C9835" + + "534E5B6E7E657765A5DFDAE2CEAF7D036ED0B7919D04FCE43CB88ECCCD995587DB30D6C7A33DC0275" + + "F1BFE615AA522826A3D4C1471A2C4A0504F0637C90F12BF811ACE90DC966D7F8E449E9712E86C22F8" + + "B959D5730FB0CA23EC0DE1814EC65433681B3095D7B30C08FF1E5804C9E725A033359E7BD6A36C598" + + "17C659E1663DF68902A35208DD1C93BF3489A02949F52CF82FF8D911A2AEB68D38E1BBA0A3BACB463" + + "FA77C24B7896FC6C9D2EBD4AE80891C8CB9973C1778B697ED68803380B8FD240C9D1450F22CA3A54A" + + "1861B65DA8E2BEDADF6AD9D8BFA66CDB2E9B58EA2C3ECB61D15581303C06092A864886F70D0107013" + + "01D060960864801650304012A0410D21007E2DFD902CBE983CAA85375891B80103B5BEA9B220BFD2C" + + "85D9AB7B153299A1A13C301C06092A864886F70D010905310F170D3236303832303139353530395A3" + + "01C06092A864886F70D010905310F170D3236303832303139353530395A").HexToByteArray(); + + EnvelopedCms ecms = new EnvelopedCms(); + ecms.Decode(encoded); + + CryptographicAttributeObjectCollection attrs = ecms.UnprotectedAttributes; + Assert.Equal(2, attrs.Count); + Assert.Equal(1, attrs[0].Values.Count); + Assert.Equal(1, attrs[1].Values.Count); + Assert.IsType(attrs[0].Values[0]); + Assert.IsType(attrs[1].Values[0]); + } + private static void AssertIsDocumentationDescription(this AsnEncodedData attribute, string expectedDocumentDescription) { Assert.Equal(Oids.DocumentDescription, attribute.Oid.Value); diff --git a/src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsTests.cs b/src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsTests.cs index c0c1421daac423..25a1a3cb7ce2fe 100644 --- a/src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsTests.cs +++ b/src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsTests.cs @@ -1611,6 +1611,159 @@ public static void Decode_CanDecodeWithAttributeCertificate() cms.CheckSignature(verifySignatureOnly: true); } + [Fact] + public static void LargeSignerCountSanityTest() + { + // Ignoring the fact that they long preceded computers, pretend the 1789 U.S. Electoral College + // signed their vote tallies with CMS. + // + // Build that, and make sure the document works at scale. + + const int ElectorsCount = 69; + X509Certificate2 root = null; + X509Certificate2 issuer = null; + X509Certificate2 lastSigner = null; + string dateSuffix = $"{DateTime.UtcNow:yyyy-MM-dd HH:mm:ss.ffff}"; + + ContentInfo content = new ContentInfo( + Text.Encoding.ASCII.GetBytes(@" +President: +George Washington (69) + +Vice President: +John Adams (34) + +No Office: +John Jay (9) +Robert H. Harrison (6) +John Rutledge (6) +John Hancock (4) +George Clinton (3) +Samuel Huntington (2) +John Milton (2) +James Armstrong (1) +Benjamin Lincoln (1) +Edward Telfair (1) +")); + + byte[] encoded; + + try + { + SignedCms builder = new SignedCms(content, detached: false); + + root = MakeRoot(dateSuffix); + issuer = MakeChild("Elector Issuing Authority", root, dateSuffix, isCA: true); + + for (int i = 0; i < ElectorsCount; i++) + { + lastSigner?.Dispose(); + lastSigner = MakeChild($"Elector {i + 1} of {ElectorsCount}", issuer, dateSuffix); + + // Alternate between IssuerAndSerialNumber and SubjectKeyIdentifier to make sure both work. + SubjectIdentifierType type = (SubjectIdentifierType)(1 + i & 1); + CmsSigner signer = new CmsSigner(type, lastSigner) + { + IncludeOption = X509IncludeOption.EndCertOnly, + }; + + if (i == 0) + { + signer.Certificates.Add(issuer); + signer.Certificates.Add(root); + } + + builder.ComputeSignature(signer, silent: true); + } + + encoded = builder.Encode(); + } + finally + { + lastSigner?.Dispose(); + issuer?.Dispose(); + root?.Dispose(); + } + + SignedCms verifier = new SignedCms(); + verifier.Decode(encoded); + + AssertExtensions.Equal(ElectorsCount + 2, verifier.Certificates.Count); + + foreach (SignerInfo signerInfo in verifier.SignerInfos) + { + Assert.NotNull(signerInfo.Certificate); + + // Assert.NoThrow + signerInfo.CheckSignature(verifySignatureOnly: true); + } + + // Assert.NoThrow + verifier.CheckSignature(verifySignatureOnly: true); + + static X509Certificate2 MakeRoot(string dateSuffix) + { + using (ECDsa key = ECDsa.Create(ECCurve.NamedCurves.nistP521)) + { + CertificateRequest req = new CertificateRequest( + $"CN=Elector Root Authority {dateSuffix}, O=Mythical Elector Attestation", + key, + HashAlgorithmName.SHA512); + + req.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, false, 0, true)); + req.CertificateExtensions.Add( + new X509SubjectKeyIdentifierExtension(req.PublicKey, false)); + DateTimeOffset now = DateTimeOffset.UtcNow; + return req.CreateSelfSigned(now.AddMinutes(-3), now.AddMinutes(120)); + } + } + + static X509Certificate2 MakeChild(string cn, X509Certificate2 issuer, string dateSuffix, bool isCA = false) + { + ECCurve curve = isCA ? ECCurve.NamedCurves.nistP384 : ECCurve.NamedCurves.nistP256; + + using (ECDsa key = ECDsa.Create(curve)) + { + CertificateRequest req = new CertificateRequest( + $"CN={cn} {dateSuffix}, O=Mythical Elector Attestation", + key, + HashAlgorithmName.SHA512); + + req.CertificateExtensions.Add(new X509BasicConstraintsExtension(isCA, false, 0, true)); + req.CertificateExtensions.Add( + new X509SubjectKeyIdentifierExtension(req.PublicKey, false)); + +#if NET + req.CertificateExtensions.Add( + X509AuthorityKeyIdentifierExtension.CreateFromCertificate(issuer, true, true)); +#endif + + byte[] serial = new byte[20]; +#if NET + RandomNumberGenerator.Fill(serial); +#else + using (RandomNumberGenerator rng = RandomNumberGenerator.Create()) + { + rng.GetBytes(serial); + } +#endif + + // Ensure positive serial number + serial[0] &= 0x7F; + + X509Certificate2 pubOnly = req.Create( + issuer, + issuer.NotBefore.AddMinutes(1), + issuer.NotAfter.AddMinutes(-1), + serial); + + X509Certificate2 withKey = pubOnly.CopyWithPrivateKey(key); + pubOnly.Dispose(); + return withKey; + } + } + } + private static void CheckNoSignature(byte[] encoded, bool badOid=false) { SignedCms cms = new SignedCms(); diff --git a/src/libraries/System.Security.Cryptography.Xml/src/Resources/Strings.resx b/src/libraries/System.Security.Cryptography.Xml/src/Resources/Strings.resx index a8cf73023e90f7..5fa8c2cb89ef23 100644 --- a/src/libraries/System.Security.Cryptography.Xml/src/Resources/Strings.resx +++ b/src/libraries/System.Security.Cryptography.Xml/src/Resources/Strings.resx @@ -1,4 +1,4 @@ - +