diff --git a/.azure-pipelines/msgraph-reference-generation.yml b/.azure-pipelines/msgraph-reference-generation.yml new file mode 100644 index 00000000..c89aab25 --- /dev/null +++ b/.azure-pipelines/msgraph-reference-generation.yml @@ -0,0 +1,164 @@ +# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the project root for license information. +# contains an end to end validation pipeline using C# compilation tests for staging beta metadata +trigger: + branches: + include: + - master + paths: + include: + - schemas/*.csdl +pr: none + +resources: + repositories: + - repository: typespec-msgraph + type: github + endpoint: microsoftgraph (22) + name: microsoftgraph/typespec-msgraph + ref: main + - repository: typespec-msgraph-reference + type: git + name: typespec-msgraph-reference +variables: + BuildConfiguration: 'Release' + scriptsDirectory: '$(Build.SourcesDirectory)\.azure-pipelines\scripts' + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: Azure-Pipelines-1ESPT-ExDShared + vmImage: windows-latest + stages: + - stage: build_reference_tool + dependsOn: [] + jobs: + - job: buildReferenceTool + displayName: "Build Reference Lib Artifact" + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: typespec-reference-tool + steps: + - task: UseDotNet@2 + displayName: 'Use .NET 10' + inputs: + version: 10.x + - checkout: typespec-msgraph-reference + displayName: checkout typespec-msgraph-reference + fetchDepth: 1 + # Install the nuget tool + - task: NuGetToolInstaller@1 + displayName: 'Install Nuget dependency manager' + inputs: + versionSpec: '>=5.2.0' + checkLatest: true + + - task: NuGetAuthenticate@1 + + # Restore dependencies + - task: DotNetCoreCLI@2 + displayName: 'Restore dependencies' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)\typespec-msgraph-reference\typespec-msgraph-reference.sln' + + # Build the solution + - task: DotNetCoreCLI@2 + displayName: 'Build solution' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\typespec-msgraph-reference\typespec-msgraph-reference.sln' + arguments: '--configuration $(BuildConfiguration) --no-incremental' + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/typespec-msgraph-reference/bin/$(BuildConfiguration)/net10.0' + contents: '**/*' + targetFolder: '$(Build.ArtifactStagingDirectory)' + displayName: Copy Typespec Reference executable + - stage: generate_reference_lib + dependsOn: [build_reference_tool] + jobs: + - job: generateLibraryFiles + displayName: "Generating reference library" + templateContext: + inputs: + - input: pipelineArtifact + buildType: 'current' + artifactName: 'typespec-reference-tool' + targetPath: '$(Build.SourcesDirectory)/typespec-reference-tool' + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/lib' + artifactName: referenceLibrary + steps: + - checkout: self # Add this to checkout msgraph-metadata repo + displayName: checkout msgraph-metadata + fetchDepth: 1 + - pwsh: '$(Build.SourcesDirectory)/.azure-pipelines/scripts/process-all-schemas.ps1 -ExecPath $(Build.SourcesDirectory)/typespec-reference-tool/typespec-msgraph-reference.exe' + displayName: "Process all the valid schemas files" + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/generated-lib/' + contents: '**/*' + targetFolder: '$(Build.ArtifactStagingDirectory)/lib' + - stage: update_reference_lib + dependsOn: [generate_reference_lib] + jobs: + - job: update_reference_lib + displayName: "Create PR to update the typespec-msgraph reference lib" + templateContext: + inputs: + - input: pipelineArtifact + buildType: 'current' + artifactName: 'referenceLibrary' + targetPath: '$(Build.SourcesDirectory)/tmp_lib' + steps: + - checkout: typespec-msgraph + displayName: checkout typespec-msgraph + fetchDepth: 1 + persistCredentials: true + - pwsh: | + git config --global user.email "GraphTooling@service.microsoft.com" + git config --global user.name "Microsoft Graph DevX Tooling" + displayName: 'Git: set user config' + # Copy files from the tmp folder to the checkout repo + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/tmp_lib' + contents: '**/*' + targetFolder: '$(Build.SourcesDirectory)/typespec-msgraph/packages/typespec-reference-library/lib' + overwrite: true + displayName: Copy OpenAPI files to local typespec-msgraph repo + # Push changes to Typespec msgraph repo + - pwsh: '$(scriptsDirectory)/git-push-reference-files.ps1' + displayName: Publish new generated files to msgraph-metadata repo + env: + PublishChanges: True + workingDirectory: '$(Build.SourcesDirectory)/typespec-msgraph' + enabled: true + + # Create PR + - task: AzureKeyVault@2 + displayName: "Azure Key Vault: Get Secrets" + inputs: + azureSubscription: "Federated AKV Managed Identity Connection" + KeyVaultName: akv-prod-eastus + SecretsFilter: "microsoft-graph-devx-bot-appid,microsoft-graph-devx-bot-privatekey" + + - pwsh: '$(scriptsDirectory)/create-pull-request.ps1' + displayName: 'Create Pull Request for the generated OpenAPI files for msgraph-metadata' + env: + BaseBranch: main + GeneratePullRequest: true + GhAppId: $(microsoft-graph-devx-bot-appid) + GhAppKey: $(microsoft-graph-devx-bot-privatekey) + OverrideSkipCI: false + RepoName: 'microsoftgraph/typespec-msgraph' + ScriptsDirectory: $(scriptsDirectory) + # Version is intentionally left empty for OpenAPI PRs as versioning is not applicable in this context. + Version: '' + workingDirectory: '$(Build.SourcesDirectory)/typespec-msgraph' + + diff --git a/.azure-pipelines/scripts/Generate-Github-Token.ps1 b/.azure-pipelines/scripts/Generate-Github-Token.ps1 new file mode 100644 index 00000000..0b1c0416 --- /dev/null +++ b/.azure-pipelines/scripts/Generate-Github-Token.ps1 @@ -0,0 +1,201 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)] + [string] + $AppClientId, + [Parameter(Mandatory = $true)] + [string] + $AppPrivateKeyContents, + [Parameter(Mandatory = $true)] + [ValidatePattern('^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$', ErrorMessage = "Repository must be in the format 'owner/repo' (e.g. 'octocat/hello-world')")] + [string] + $Repository +) + +$ErrorActionPreference = "Stop" + +function Generate-AppToken { + param ( + [string] + $ClientId, + [string] + $PrivateKeyContents + ) + + $header = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + alg = "RS256" + typ = "JWT" + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + $payload = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((ConvertTo-Json -InputObject @{ + iat = [System.DateTimeOffset]::UtcNow.AddSeconds(-10).ToUnixTimeSeconds() + exp = [System.DateTimeOffset]::UtcNow.AddMinutes(1).ToUnixTimeSeconds() + iss = $ClientId + }))).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + $rsa = [System.Security.Cryptography.RSA]::Create() + $rsa.ImportFromPem($PrivateKeyContents) + + $signature = [Convert]::ToBase64String($rsa.SignData([System.Text.Encoding]::UTF8.GetBytes("$header.$payload"), [System.Security.Cryptography.HashAlgorithmName]::SHA256, [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)).TrimEnd('=').Replace('+', '-').Replace('/', '_') + $jwt = "$header.$payload.$signature" + + return $jwt +} + +function Generate-InstallationToken { + param ( + [string] + $AppToken, + [string] + $InstallationId, + [string] + $Repository + ) + + $uri = "https://api.github.com/app/installations/$InstallationId/access_tokens" + $headers = @{ + Authorization = "Bearer $AppToken" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + + $body = @{ + repositories = @($Repository) + } + + $response = Invoke-RestMethod -Uri $uri -Method Post -Headers $headers -Body (ConvertTo-Json -InputObject $body -Compress -Depth 10) + + return $response.token +} + + +function Get-OrganizationInstallationId { + param ( + [string] + $AppToken, + [string] + $Organization + ) + + $uri = "https://api.github.com/orgs/$Organization/installation" + $headers = @{ + Authorization = "Bearer $AppToken" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + + try { + $response = Invoke-RestMethod -Uri $uri -Method Get -Headers $headers + + return $response.id + } + catch [Microsoft.PowerShell.Commands.HttpResponseException] { + if ($_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::UnprocessableContent -or $_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::NotFound) { + return $null + } + + throw + } +} + +function Get-RepositoryInstallationId { + param ( + [string] + $AppToken, + [string] + $Repository + ) + + $uri = "https://api.github.com/repos/$Repository/installation" + $headers = @{ + Authorization = "Bearer $AppToken" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + + try { + $response = Invoke-RestMethod -Uri $uri -Method Get -Headers $headers + + return $response.id + } + catch [Microsoft.PowerShell.Commands.HttpResponseException] { + if ($_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::UnprocessableContent -or $_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::NotFound) { + return $null + } + + throw + } +} + +function Get-UserInstallationId { + param ( + [string] + $AppToken, + [string] + $Username + ) + + $uri = "https://api.github.com/users/$Username/installation" + $headers = @{ + Authorization = "Bearer $AppToken" + Accept = "application/vnd.github+json" + "X-GitHub-Api-Version" = "2022-11-28" + } + + try { + $response = Invoke-RestMethod -Uri $uri -Method Get -Headers $headers + + return $response.id + } + catch [Microsoft.PowerShell.Commands.HttpResponseException] { + if ($_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::UnprocessableContent -or $_.Exception.Response.StatusCode -eq [System.Net.HttpStatusCode]::NotFound) { + return $null + } + + throw + } +} + +function Get-InstallationId { + param ( + [string] + $AppToken, + [string] + $Owner, + [string] + $Repo + ) + + $orgInstallationId = Get-OrganizationInstallationId -AppToken $AppToken -Organization $Owner + + if ($null -eq $orgInstallationId) { + $repoInstallationId = Get-RepositoryInstallationId -AppToken $AppToken -Repository "$Owner/$Repo" + } + else { + return $orgInstallationId + } + + if ($null -eq $repoInstallationId) { + $userInstallationId = Get-UserInstallationId -AppToken $AppToken -Username $Owner + } + else { + return $repoInstallationId + } + + if ($null -eq $userInstallationId) { + throw "Installation not found for repository '$Repo'" + } + else { + return $userInstallationId + } +} + +$owner, $repo = $Repository -split '/' + +$AppToken = Generate-AppToken -ClientId $AppClientId -PrivateKeyContents $AppPrivateKeyContents + +$InstallationId = Get-InstallationId -AppToken $AppToken -Owner $owner -Repo $repo + +$InstallationToken = Generate-InstallationToken -AppToken $AppToken -InstallationId $InstallationId -Repository $repo + +Write-Output $InstallationToken \ No newline at end of file diff --git a/.azure-pipelines/scripts/create-pull-request.ps1 b/.azure-pipelines/scripts/create-pull-request.ps1 new file mode 100644 index 00000000..95279441 --- /dev/null +++ b/.azure-pipelines/scripts/create-pull-request.ps1 @@ -0,0 +1,24 @@ + +if (($env:GeneratePullRequest -eq $False)) { # Skip CI if manually running this pipeline. + Write-Host "Skipping pull request creation due this repository being disabled" + return; +} + +$title = "Generated $env:Version typespec reference files" + + +$body = ":bangbang:**_Important_**:bangbang:
Check for unexpected deletions or changes in this PR and ensure relevant CI checks are passing.

**Note:** This pull request was automatically created by Azure pipelines." + +# The installed application is required to have the following permissions: read/write on pull requests/ +$tokenGenerationScript = "$env:ScriptsDirectory\Generate-Github-Token.ps1" +$env:GITHUB_TOKEN = & $tokenGenerationScript -AppClientId $env:GhAppId -AppPrivateKeyContents $env:GhAppKey -Repository $env:RepoName +Write-Host "Fetched Github Token for PR generation and set as environment variable." + +# No need to specify reviewers as code owners should be added automatically. +if (![string]::IsNullOrEmpty($env:BaseBranch)) { + gh pr create -t $title -b $body -B $env:BaseBranch +} else { + gh pr create -t $title -b $body +} + +Write-Host "Pull Request Created successfully." \ No newline at end of file diff --git a/.azure-pipelines/scripts/git-push-reference-files.ps1 b/.azure-pipelines/scripts/git-push-reference-files.ps1 new file mode 100644 index 00000000..1e93de74 --- /dev/null +++ b/.azure-pipelines/scripts/git-push-reference-files.ps1 @@ -0,0 +1,58 @@ +# Based on git-push-cleanmetadata.ps1 in the code generator repo +# Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License in the +# project root for license information. + +# This script stashes any changes, checks out the latest main branch, applies the stashed changes, commits, and +# pushes the changes back to the remote repository. + +Write-Host "`n1. git status:" +git status | Write-Host + +Write-Host "`n2. Stash the update reference library files.....`n3. Running: git stash" +git stash | Write-Host + +Write-Host "`n4. Fetching latest main branch to ensure we are up to date..." +git fetch origin main | Write-Host +# checkout main to move from detached HEAD mode +git switch main | Write-Host + + + +Write-Host "`n5. git status:" +git status | Write-Host + +Write-Host "`n6. Apply stashed reference library files...`n7. Running: git stash pop" +git stash pop | Write-Host + +Write-Host "`n8. git status:" +git status | Write-Host + +$branch = "update-reference-library-files/$env:BUILD_BUILDID" +Write-Host "`n9. Create branch: $branch" +git checkout -B $branch | Write-Host + + +Write-Host "`n10. Staging reference library files.....`n11. Running: git add ." +git add . | Write-Host + +Write-Host "`n12. git status:" +git status | Write-Host + +Write-Host "`n13. Attempting to commit clean reference library files....." + +if ($env:BUILD_REASON -eq 'Manual') # Skip CI if manually running this pipeline. +{ + git commit -m "Update reference library files with $env:BUILD_BUILDID [skip ci]" | Write-Host +} +else +{ + git commit -m "Update reference library files with $env:BUILD_BUILDID" | Write-Host +} + +Write-Host "`n14. git status:" +git status | Write-Host + +Write-Host "`n15a. Pushing branch for PR creation" + +Write-Host "`n15b. Running: git push --set-upstream origin $branch" +git push --set-upstream origin $branch | Write-Host diff --git a/.azure-pipelines/scripts/process-all-schemas.ps1 b/.azure-pipelines/scripts/process-all-schemas.ps1 new file mode 100644 index 00000000..5ac11eb6 --- /dev/null +++ b/.azure-pipelines/scripts/process-all-schemas.ps1 @@ -0,0 +1,61 @@ +# Process all CSDL files with typespec-msgraph-reference.exe +# This script processes both beta and v1.0 CSDL files for all environments + +param( + [Parameter(Mandatory=$true)] + [string]$ExePath +) + +$ErrorActionPreference = "Stop" + +# Define the environments +$environments = @( + "Bleu", + "Delos", + "Fairfax", + "GovSG", + "Mooncake", + "Prod", + "USNat", + "USSec" +) + +# Process beta files +Write-Host "Processing beta CSDL files..." -ForegroundColor Cyan +foreach ($env in $environments) { + $csdlFile = "../../schemas/beta-$env.csdl" + $outputDir = "../../generated-lib/$env/Beta/" + + if (Test-Path $csdlFile) { + Write-Host "Processing $csdlFile -> $outputDir" -ForegroundColor Green + & $ExePath $csdlFile $outputDir + + if ($LASTEXITCODE -ne 0) { + Write-Host "Error processing $csdlFile (Exit code: $LASTEXITCODE)" -ForegroundColor Red + exit $LASTEXITCODE + } + } else { + Write-Host "Warning: $csdlFile not found, skipping..." -ForegroundColor Yellow + } +} + +# Process v1.0 files +Write-Host "`nProcessing v1.0 CSDL files..." -ForegroundColor Cyan +foreach ($env in $environments) { + $csdlFile = "../../schemas/v1.0-$env.csdl" + $outputDir = "../../generated-lib/$env/V1.0/" + + if (Test-Path $csdlFile) { + Write-Host "Processing $csdlFile -> $outputDir" -ForegroundColor Green + & $ExePath $csdlFile $outputDir + + if ($LASTEXITCODE -ne 0) { + Write-Host "Error processing $csdlFile (Exit code: $LASTEXITCODE)" -ForegroundColor Red + exit $LASTEXITCODE + } + } else { + Write-Host "Warning: $csdlFile not found, skipping..." -ForegroundColor Yellow + } +} + +Write-Host "`nAll CSDL files processed successfully!" -ForegroundColor Green