azure-pipelines-bulk-data.yml

trigger: none

pool:
  vmImage: "ubuntu-latest"

parameters:
  - name: env
    displayName: "Environment to import data into"
    type: string
    default: ""
    values: ["dev", "int", "stag"]
  - name: csvFileName
    displayName: "Filename of the CSV file to import"
    type: string
    default: ""
  - name: importItemType
    displayName: "Select type of data to import:"
    type: string
    default: ""
    values: ["site", "user", "apiUser"]

variables:
  isMain: ${{eq(variables['Build.SourceBranch'], 'refs/heads/main')}}
  environment: ${{parameters.env}}
  csvFile: ${{parameters.csvFileName}}
  importItemType: ${{parameters.importItemType}}
  storageAccountName: "nbsmyastrgimportsintuks"
  containerName: "csvdata"
  cosmosCoreDataPath: "data/CosmosDbSeeder/items/$(environment)/core_data"
  cosmosAccountName: "nbs-mya-cdb-$(environment)-uks"
  resourceGroupName: "nbs-mya-rg-$(environment)-uks"

stages:
  - stage: "BulkDataImport"
    displayName: "Bulk Data Import"
    jobs:
      - job: "BulkDataImport"
        displayName: "Bulk Data Import"
        condition: eq(variables.isMain, true)
        steps:
          - pwsh: |
              Remove-Item '$(cosmosCoreDataPath)/*'
            displayName: "Remove existing cosmos files"
          - task: AzureCLI@2
            displayName: "Download CSV file"
            inputs:
              azureSubscription: "nbs-mya-rg-int"
              scriptType: pscore
              scriptLocation: scriptPath
              scriptPath: "$(Build.SourcesDirectory)/scripts/storage-account/download_blob.ps1"
              arguments: "-storageAccountName $(storageAccountName) -containerName $(containerName) -blobName $(csvFile) -outputFilePath $(csvFile)"
          - task: DotNetCoreCLI@2
            displayName: "Run CSV conversion tool"
            inputs:
              command: "run"
              projects: "data/CsvDataTool/CsvDataTool.csproj"
              arguments: "--in $(csvFile) --out $(cosmosCoreDataPath) --itemType $(importItemType)"
          - task: PublishBuildArtifacts@1
            displayName: "Publish CSV conversion report"
            inputs:
              PathtoPublish: "csv_conversion_report.md"
              ArtifactName: "CSV Conversion Report"
          - pwsh: |
              Write-Host "##vso[task.uploadsummary]$(Build.SourcesDirectory)/csv_conversion_report_summary.md"
            displayName: "Attach CSV conversion summary report"
          - task: AzureCLI@2
            displayName: "Get Cosmos DB account connection string"
            inputs:
              azureSubscription: "nbs-mya-rg-$(environment)"
              scriptType: pscore
              scriptLocation: scriptPath
              scriptPath: "$(Build.SourcesDirectory)/scripts/cosmos-build/get_cosmos_connection_string.ps1"
              arguments: "-resourceGroup $(resourceGroupName) -cosmosAccountName $(cosmosAccountName)"
          - task: DotNetCoreCLI@2
            displayName: "Seed Cosmos DB containers"
            inputs:
              command: "run"
              projects: "data/CosmosDbSeeder/CosmosDbSeeder.csproj"
              arguments: "--no-overwrites"
            env:
              COSMOS_ENDPOINT: $(COSMOS_ENDPOINT)
              COSMOS_TOKEN: $(COSMOS_TOKEN)
          - task: PublishBuildArtifacts@1
            displayName: "Publish full data import report"
            inputs:
              PathtoPublish: "data_import_report.md"
              ArtifactName: "Data Import Report"
          - pwsh: |
              Write-Host "##vso[task.uploadsummary]$(Build.SourcesDirectory)/data_import_summary.md"
            displayName: "Attach data import summary report"