diff --git a/.config/LocalizationValidationAllowlist.json b/.config/LocalizationValidationAllowlist.jsonc similarity index 73% rename from .config/LocalizationValidationAllowlist.json rename to .config/LocalizationValidationAllowlist.jsonc index 0a3cbfea6b..58dfd5e357 100644 --- a/.config/LocalizationValidationAllowlist.json +++ b/.config/LocalizationValidationAllowlist.jsonc @@ -1,9 +1,19 @@ { - "_comment": [ - "These culture/key pairs intentionally match the English source text.", - "Each pair was verified in the internal LCL source as localized (Stat=Loc, Orig=New). Remove an entry when its localized value changes." - ], - "AllowedEnglishValueMatches": { + // Schema: { "": { "": ["", ...] } } + // Owners: SqlClient (driver resources) and Abstractions (authentication resources). + // Each owner maps Strings..resx filenames to arrays of approved resource keys. + // An empty owner object means no exceptions. Omit files that need no exceptions. + // The validator reads only the selected owner; a missing owner section is an error. + // Filenames and keys are case-sensitive and must exist in the selected resource set. + // Keys must be non-empty, unique within each file's array, and have non-empty English values. + // An exception permits only an exact English-value match in that one owner and culture. + // It cannot exempt missing keys, empty translations, or keys absent from Strings.resx. + // + // These culture/key pairs intentionally match the English source text. + // Each pair was verified in the internal LCL source as localized (Stat=Loc, Orig=New). + // Remove an entry when its localized value changes. + "Abstractions": {}, + "SqlClient": { "Strings.cs.resx": [ "ADP_InvalidMultipartName", "DataCategory_Data", diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 83826954bd..e387af46fb 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -30,7 +30,7 @@ Respect this graph when modifying build stages: Validation runs in three places and shares one gating switch. -- **Localization** — `steps/validate-localization-step.yml`, in the SqlClient build job before the driver is built. Reports missing or obsolete keys, empty localized values whose English value is non-empty, and untranslated resources. Approved identical translations are listed by culture and resource key in `.config/LocalizationValidationAllowlist.json`. +- **Localization** — `steps/validate-localization-step.yml`, in the SqlClient and Abstractions build jobs before compilation. Callers explicitly supply the resource directory and allowlist section. Reports missing or obsolete keys, empty localized values whose English value is non-empty, and untranslated resources. Approved identical translations are nested by resource owner, culture, and resource key in `.config/LocalizationValidationAllowlist.jsonc`: `SqlClient` for the driver and `Abstractions` for Abstractions. Use JSON comments to document exceptions. - **XML documentation** — `eng/pipelines/common/steps/validate-xml-docs-step.yml`, shared with PR validation, in three modes: snippet sources before each snippet-consuming project is built (SqlClient, SqlServer, Abstractions, and Azure), generated documentation after documented projects are built (including the separate SqlClient reference-assembly output), and assembled packages in `package_validation`. Reports malformed documentation IDs, unresolved cross-references, and `lib/` vs `ref/` documentation-trimming defects. - **Packages** — `steps/validate-packages-step.yml`, in `package_validation`. Runs `tools/PackageValidator` across the whole drop so its cross-package version and dependency rules apply. diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index ea263503f7..7020d4c2c9 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -495,6 +495,53 @@ request submission, not symbol availability. With the default setting, the existing status check rejects terminal failures but does not wait for pending requests to complete. +## Localization Validation + +OneBranch validates localized resources before building the SqlClient and Abstractions +assemblies. Each caller explicitly supplies its resource directory and isolated allowlist +section in [LocalizationValidationAllowlist.jsonc](.config/LocalizationValidationAllowlist.jsonc): +`SqlClient` for the driver and `Abstractions` for Abstractions. Each owner contains +localized filenames mapped to lists of resource keys approved to match English. +The script defaults to `SqlClient` for direct callers. +The JSONC file supports comments for documenting approved exceptions. + +### Allowlist schema + +The file is a JSON object with three levels: + +1. **Resource owner**: `SqlClient` or `Abstractions`, selecting an independent exception set. +2. **Localized resource filename**: `Strings..resx`, relative to that owner's + resource directory. Filenames are case-sensitive. +3. **Resource keys**: an array of case-sensitive key names approved to have localized + values exactly equal to their non-empty English values in `Strings.resx`. + +For example, this abbreviated configuration permits `DataCategory_Xml` to remain +identical to English in the driver's French resources, but grants no Abstractions exceptions: + +```jsonc +{ + "SqlClient": { + // Approved for this resource owner and culture only. + "Strings.fr.resx": ["DataCategory_Xml"] + }, + "Abstractions": {} +} +``` + +Use an empty object (`{}`) for an owner with no exceptions, and omit filenames that need +none. The selected owner section must exist; the validator never borrows another owner's +exceptions. Each filename must exist in the selected resource directory. Each listed key +must be non-empty, unique within its array, and present with a non-empty English value. + +Exceptions permit English-value matches only: they do not excuse missing resource keys, +empty translations, or keys absent from the English resource file. Remove an exception +when its localized value changes; stale exceptions are reported as validation findings. +Only add verified, intentional English matches, not strings still awaiting translation. + +The existing `failOnValidationError` flag controls whether findings fail the build or +are reported as warnings. Missing or malformed validator inputs always fail. Do not +allowlist untranslated development strings merely to make validation pass. + ## Versioning Versioning can be accomplished by using a mix of different parameters to the `build.proj` targets: diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index 00c3900b55..84685a3b88 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -153,11 +153,19 @@ jobs: steps: - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self - # Localized resources ship with the SqlClient driver. Validate them before analysis and - # building so missing or untranslated strings fail every SqlClient build. + # Validate each resource-owning assembly before analysis and building. - ${{ if eq(parameters.packageShortName, 'SqlClient') }}: - template: /eng/pipelines/onebranch/steps/validate-localization-step.yml@self parameters: + resourcesDirectory: src/Microsoft.Data.SqlClient/src/Resources + allowlistSection: SqlClient + failOnValidationError: ${{ parameters.failOnValidationError }} + + - ${{ if eq(parameters.packageShortName, 'Abstractions') }}: + - template: /eng/pipelines/onebranch/steps/validate-localization-step.yml@self + parameters: + resourcesDirectory: src/Microsoft.Data.SqlClient.Extensions/Abstractions/src/Resources + allowlistSection: Abstractions failOnValidationError: ${{ parameters.failOnValidationError }} # Validate the documentation snippet sources. This requires no build, so a malformed diff --git a/eng/pipelines/onebranch/scripts/tests/pipeline-invocation.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/pipeline-invocation.Tests.ps1 index dd58c49ee5..a8518e5830 100644 --- a/eng/pipelines/onebranch/scripts/tests/pipeline-invocation.Tests.ps1 +++ b/eng/pipelines/onebranch/scripts/tests/pipeline-invocation.Tests.ps1 @@ -49,6 +49,24 @@ BeforeAll { } Describe 'Validation step templates' { + It 'selects the resource owner allowlist for ' -ForEach @( + @{ Owner = 'SqlClient'; Directory = 'src/Microsoft.Data.SqlClient/src/Resources' } + @{ Owner = 'Abstractions'; Directory = 'src/Microsoft.Data.SqlClient.Extensions/Abstractions/src/Resources' } + ) { + $content = Get-Content -LiteralPath (Join-Path $script:jobsPath 'build-buildproj-job.yml') -Raw + $allowlistPath = Join-Path $script:repoRoot '.config/LocalizationValidationAllowlist.jsonc' + $allowlist = Get-Content -LiteralPath $allowlistPath -Raw | ConvertFrom-Json + $step = Get-Content -LiteralPath ( + Join-Path $script:stepsPath 'validate-localization-step.yml') -Raw + + $step | Should -Match '\.config/LocalizationValidationAllowlist\.jsonc"' + $content | Should -Match ( + 'resourcesDirectory: ' + [regex]::Escape($Directory) + + '\r?\n\s+allowlistSection: ' + [regex]::Escape($Owner) + '\r?\n') + $allowlist.PSObject.Properties.Name | Should -Contain $Owner + Test-Path -LiteralPath (Join-Path $script:repoRoot $Directory) | Should -BeTrue + } + It 'never passes a switch using the -Switch:Value form' -ForEach @( @{ Template = 'validate-localization-step.yml' } @{ Template = 'validate-packages-step.yml' } diff --git a/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 index 4b1903279d..9c0806b72c 100644 --- a/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 +++ b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 @@ -36,6 +36,70 @@ BeforeAll { } Describe 'validate-localization.ps1' { + It 'accepts JSON comments in the allowlist' -ForEach @( + @{ Owner = 'SqlClient' } + @{ Owner = 'Abstractions' } + ) { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + $allowlist = Join-Path $resources 'allowlist.jsonc' + @' +{ + // Approved English matches, isolated by resource owner. + "SqlClient": { "Strings.fr.resx": ["Greeting"] }, + /* Abstractions has an independent set of approved matches. */ + "Abstractions": { "Strings.fr.resx": ["Greeting"] } +} +'@ | Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist ` + -AllowlistSection $Owner } | Should -Not -Throw + } + + It 'selects an independent allowlist for a second resource assembly' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + $allowlist = Join-Path $resources 'allowlist.json' + @{ + SqlClient = @{ 'Strings.fr.resx' = @('UnrelatedDriverKey') } + Abstractions = @{} + } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $allowlist + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist ` + -AllowlistSection Abstractions } | Should -Not -Throw + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist ` + -AllowlistSection SqlClient } | + Should -Throw '*Localization validation failed with 1 error*' + } + + It 'rejects a missing resource-set allowlist section rather than borrowing another assembly allowlist' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + $allowlist = Join-Path $resources 'allowlist.json' + @{ SqlClient = @{} } | ConvertTo-Json | Set-Content -LiteralPath $allowlist + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist ` + -AllowlistSection Abstractions } | + Should -Throw "*must define 'Abstractions'*" + } + + It 'honors an English-match exception only in the selected resource-set section' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + $allowlist = Join-Path $resources 'allowlist.json' + @{ + SqlClient = @{} + Abstractions = @{ 'Strings.fr.resx' = @('Greeting') } + } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist ` + -AllowlistSection Abstractions } | Should -Not -Throw + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error*' + } + It 'accepts complete localized files with translated values' { $resources = New-ResourcesDirectory Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } @@ -105,7 +169,7 @@ Describe 'validate-localization.ps1' { $allowlist = Join-Path $resources 'allowlist.json' Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } - @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + @{ SqlClient = @{ 'Strings.fr.resx' = @('Greeting') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist @@ -118,7 +182,7 @@ Describe 'validate-localization.ps1' { $allowlist = Join-Path $resources 'allowlist.json' Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } - @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Farewell') } } | + @{ SqlClient = @{ 'Strings.fr.resx' = @('Farewell') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist @@ -131,7 +195,7 @@ Describe 'validate-localization.ps1' { $allowlist = Join-Path $resources 'allowlist.json' Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } - @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unknown') } } | + @{ SqlClient = @{ 'Strings.fr.resx' = @('Unknown') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist @@ -145,7 +209,7 @@ Describe 'validate-localization.ps1' { Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hello' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } - @{ AllowedEnglishValueMatches = @{ 'Strings.de.resx' = @('Greeting') } } | + @{ SqlClient = @{ 'Strings.de.resx' = @('Greeting') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist @@ -167,7 +231,7 @@ Describe 'validate-localization.ps1' { $allowlist = Join-Path $resources 'allowlist.json' Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } - @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + @{ SqlClient = @{ 'Strings.fr.resx' = @('Greeting') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist @@ -180,7 +244,7 @@ Describe 'validate-localization.ps1' { $allowlist = Join-Path $resources 'allowlist.json' Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Unused = '' } - @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unused') } } | + @{ SqlClient = @{ 'Strings.fr.resx' = @('Unused') } } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $allowlist diff --git a/eng/pipelines/onebranch/scripts/validate-localization.ps1 b/eng/pipelines/onebranch/scripts/validate-localization.ps1 index 6cc9b59099..3badcf0394 100644 --- a/eng/pipelines/onebranch/scripts/validate-localization.ps1 +++ b/eng/pipelines/onebranch/scripts/validate-localization.ps1 @@ -6,11 +6,16 @@ Directory containing the English and localized Strings.resx files. .PARAMETER AllowlistPath - Optional JSON file containing approved English-value matches grouped by localized filename. + Optional JSONC file containing approved English-value matches grouped by resource owner, + localized filename, and resource key. .PARAMETER ReportOnly Report validation findings as warnings without failing. Malformed or missing inputs still fail, because a run that could not examine the resources has produced no result to report. + +.PARAMETER AllowlistSection + Resource owner containing the allowlist for the resource set being validated. + Defaults to SqlClient. #> # Licensed to the .NET Foundation under one or more agreements. @@ -25,7 +30,9 @@ param( [string]$AllowlistPath, - [switch]$ReportOnly + [switch]$ReportOnly, + + [string]$AllowlistSection = 'SqlClient' ) Set-StrictMode -Version Latest @@ -83,9 +90,9 @@ if (-not [string]::IsNullOrWhiteSpace($AllowlistPath)) { } $configuration = Get-Content -LiteralPath $AllowlistPath -Raw | ConvertFrom-Json - $englishValueMatchesProperty = $configuration.PSObject.Properties['AllowedEnglishValueMatches'] + $englishValueMatchesProperty = $configuration.PSObject.Properties[$AllowlistSection] if ($null -eq $englishValueMatchesProperty) { - throw "Localization allowlist file '$AllowlistPath' must define 'AllowedEnglishValueMatches'." + throw "Localization allowlist file '$AllowlistPath' must define '$AllowlistSection'." } foreach ($fileProperty in $englishValueMatchesProperty.Value.PSObject.Properties) { diff --git a/eng/pipelines/onebranch/steps/validate-localization-step.yml b/eng/pipelines/onebranch/steps/validate-localization-step.yml index 7036eaff55..830bd97447 100644 --- a/eng/pipelines/onebranch/steps/validate-localization-step.yml +++ b/eng/pipelines/onebranch/steps/validate-localization-step.yml @@ -9,6 +9,11 @@ parameters: # the step succeeds. - name: failOnValidationError type: boolean + # Each resource owner supplies its directory and isolated exception set explicitly. + - name: resourcesDirectory + type: string + - name: allowlistSection + type: string steps: # Compose the argument line first so the optional switch can be appended without repeating the @@ -21,8 +26,9 @@ steps: # other than True or False is rejected rather than assumed, so an unrecognised value cannot # silently downgrade validation to warnings. - pwsh: | - $arguments = '-ResourcesDirectory "$(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/src/Resources"' + - ' -AllowlistPath "$(Build.SourcesDirectory)/.config/LocalizationValidationAllowlist.json"' + $arguments = '-ResourcesDirectory "$(Build.SourcesDirectory)/${{ parameters.resourcesDirectory }}"' + + ' -AllowlistPath "$(Build.SourcesDirectory)/.config/LocalizationValidationAllowlist.jsonc"' + + ' -AllowlistSection "${{ parameters.allowlistSection }}"' $failOnValidationError = '${{ parameters.failOnValidationError }}' if ($failOnValidationError -eq 'False') { diff --git a/eng/pipelines/onebranch/steps/validate-packages-step.yml b/eng/pipelines/onebranch/steps/validate-packages-step.yml index e66756368f..5d256bf21e 100644 --- a/eng/pipelines/onebranch/steps/validate-packages-step.yml +++ b/eng/pipelines/onebranch/steps/validate-packages-step.yml @@ -59,6 +59,9 @@ parameters: type: boolean steps: + - task: NuGetAuthenticate@1 + displayName: 'Authenticate NuGet feeds' + - task: DotNetCoreCLI@2 displayName: 'build.proj - BuildPackageValidator' inputs: