Report cypress test results into testrail. (#9685)

* Reduce cypress size/scope

* Emit cypress reports in junit-compatible format.

* Clean implementation of upload

* Append a warning if not all tests that testrail knows of are run.

* Only run testrail if on develop (like percy).

* Re-enable full test run.

* Update cypress.config.ts

* Ensure SUITE_ID is passed.

* Fix quoting on SUITE_ID, add environment

* Update .github/workflows/cypress.yaml

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

* re-add mocha-junit-reporter (was dropped in the merge)

* Prettifier on newly created file.

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

.github/workflows/cypress.yaml

@@ -22,6 +22,7 @@ jobs:
             commit_author: ${{ steps.commit.outputs.author }}
             commit_email: ${{ steps.commit.outputs.email }}
             percy_enable: ${{ steps.percy.outputs.value || '1' }}
+            testrail_enable: ${{ steps.testrail.outputs.value || '1' }}
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
             # This provides an easy link to this workflow_run from the PR before Cypress is done.
@@ -63,6 +64,14 @@ jobs:
                   !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy')
               run: echo "::set-output name=value::0"
 
+            # Only run Testrail when it is demanded or on develop
+            - name: Disable Testrail if not needed
+              id: testrail
+              if: |
+                  github.event.workflow_run.event == 'pull_request' &&
+                  !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Send-Testrail')
+              run: echo "::set-output name=value::0"
+
             - name: Generate unique ID 💎
               id: uuid
               run: echo "::set-output name=value::sha-$GITHUB_SHA-time-$(date +"%s")"
@@ -114,6 +123,7 @@ jobs:
                       #parallel: true
                       #command-prefix: 'yarn percy exec --parallel --'
                   command-prefix: "yarn percy exec --"
+                  config: '{"reporter":"cypress-multi-reporters", "reporterOptions": { "configFile": "cypress-ci-reporter-config.json" } }'
                   ci-build-id: ${{ needs.prepare.outputs.uuid }}
               env:
                   # pass the Dashboard record key as an environment variable
@@ -158,6 +168,12 @@ jobs:
                       cypress/videos
                       cypress/synapselogs
 
+            - name: Upload reports
+              if: always()
+              uses: actions/upload-artifact@v2
+              with:
+                  name: cypress-junit
+                  path: cypress/results
     report:
         name: Report results
         needs: tests
@@ -173,3 +189,68 @@ jobs:
                   context: ${{ github.workflow }} / cypress (${{ github.event.workflow_run.event }} => ${{ github.event_name }})
                   sha: ${{ github.event.workflow_run.head_sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    testrail:
+        name: Report results to testrail
+        needs:
+            - prepare
+            - tests
+        environment: Testrail
+        runs-on: ubuntu-latest
+        if: ${{ needs.prepare.outputs.testrail_enable }}
+        steps:
+            - uses: actions/download-artifact@v3
+              with:
+                  name: cypress-junit
+            - name: Prepare testrail upload config
+              id: testrailprep
+              env:
+                  TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
+                  TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
+                  TEST_ID: ${{ github.run_id }}
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+              run: |
+                  echo '{"name": "element-web cypress '$TEST_ID'", "suite_id": '$TESTRAIL_SUITE_ID' }' > body.json # TODO add description with more context?
+                  RUN_ID=`curl -X POST -d @body.json -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/add_run/$TESTRAIL_PROJECT_ID" | jq '.id'`
+                  PROJECT_NAME=`curl -X GET -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/get_project/$TESTRAIL_PROJECT_ID" | jq '.name'`
+                  echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
+                  echo "project_name=$PROJECT_NAME" >> $GITHUB_OUTPUT
+            - name: setup python
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.11"
+            - run: pip install trcli
+            - name: Upload junit files
+              env:
+                  TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
+                  TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+                  TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
+              run: |
+                  for file in results-*.xml; do
+                      echo "Handling $file"
+                      trcli -y -h $TESTRAIL_URL \
+                                  --project-id $TESTRAIL_PROJECT_ID \
+                                  --project ${{ steps.testrailprep.outputs.project_name }} \
+                                  --username $TESTRAIL_USER \
+                                  --password $TESTRAIL_API_KEY \
+                                  parse_junit \
+                                  --run-id $TESTRAIL_RUN_ID \
+                                  --suite-id $TESTRAIL_SUITE_ID \
+                                  --title "if you see this please check cypress build for run id not being provisioned" \
+                                  -f $file
+                  done
+            - name: Close test run
+              id: testrailpost
+              if: always()
+              env:
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+                  TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
+              run: |
+                  CLOSE_RESPONSE=`curl -X POST -d '{}' -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/close_run/$TESTRAIL_RUN_ID"`
+                  if [ ! "0" == "`echo $CLOSE_RESPONSE | jq .untested_count`" ] ; then echo "::warning title=Missing Tests::Testrail reported some cypress tests were not run. $CLOSE_RESPONSE"; fi