Merge pull request North-Seattle-College#129 from gillilands19/SG-artillery-perf

[Task] Create and Deploy Serverless Artillery test for /Users/{UserID}/Tasks/{TaskID}

Author: gillilands19

.gitignore

@@ -117,3 +117,6 @@ DS_Store
 
 # ignore .vscode
 .vscode/
+
+package-lock.json
+

package-lock.json

@@ -0,0 +1 @@
+version: 0.0.1

test/artillery/.slsart

@@ -0,0 +1,50 @@
+const aws = require('aws-sdk') // eslint-disable-line import/no-extraneous-dependencies
+
+const sns = new aws.SNS()
+
+const alert = {
+  briefAnalysis: (analysis) => {
+    const latencies = []
+    analysis.reports.forEach((report, i) => {
+      const current = report
+      latencies[i] = current.latencies
+      delete current.latencies
+    })
+    const ret = JSON.stringify(analysis, null, 2)
+    analysis.reports.forEach((report, i) => {
+      const current = report
+      current.latencies = latencies[i]
+    })
+    return ret
+  },
+  send: (script, analysis) => {
+    if (!process.env.TOPIC_ARN) {
+      console.error([
+        '#########################################################',
+        '##         ! Required Configuration Missing !          ##',
+        '## in order to send the alert, an environment variable ##',
+        '## TOPIC_ARN must be available.  An alert was supposed ##',
+        '## to be sent but one cannot be sent!                  ##',
+        '#########################################################',
+      ].join('\n'))
+      return Promise.resolve()
+    } else {
+      const subject = `Alert: ${analysis.errorMessage}`
+      const message = `Alert:
+  ${analysis.errorMessage}
+
+Logs:
+Full analysis:
+${alert.briefAnalysis(analysis)}
+`
+      const params = {
+        Subject: subject,
+        Message: message,
+        TopicArn: process.env.TOPIC_ARN,
+      }
+      return sns.publish(params).promise()
+    }
+  },
+}
+
+module.exports = alert

test/artillery/alert.js

@@ -0,0 +1,131 @@
+const modes = require('./modes.js')
+const sampling = require('./sampling.js')
+
+const analysis = {
+  getErrorBudget: (script, defaultErrorBudget) => {
+    if (script && script.sampling && script.sampling.errorBudget !== undefined) {
+      return script.sampling.errorBudget
+    } else if (defaultErrorBudget !== undefined) {
+      return defaultErrorBudget
+    } else {
+      return sampling.defaults.sampling.errorBudget
+    }
+  },
+
+  /**
+   * Analyze a set of sampling reports, each of which is the result of a sample battery's execution
+   * results.
+   * @param script Artillery script used
+   * @param report The collection of reports to analyze
+   * @param defaultErrorBudget Maximum number of errors allowed
+   */
+  analyzeSamples: (script, report, defaultErrorBudget) => {
+    let reports
+
+    if (report.reports) {
+      reports = report.reports // eslint-disable-line prefer-destructuring
+    } else {
+      reports = [report]
+    }
+
+    const totals = {
+      scenariosCreated: 0,
+      scenariosCompleted: 0,
+      requestsCompleted: 0,
+      codes: {},
+      errors: {},
+    }
+
+    const mergeReportPropertyCounts = property =>
+      reports.forEach(test =>
+        Object.keys(test[property]).forEach((key) => {
+          const countForTest = test[property][key]
+          const isNewKey = totals[property][key] === undefined
+
+          if (isNewKey) {
+            totals[property][key] = countForTest
+          } else {
+            totals[property][key] += countForTest
+          }
+        }))
+
+    mergeReportPropertyCounts('codes')
+    mergeReportPropertyCounts('errors')
+
+    const accumulateReportTotals = property =>
+      reports.forEach((test) => { totals[property] += test[property] })
+
+    accumulateReportTotals('scenariosCreated')
+    accumulateReportTotals('scenariosCompleted')
+    accumulateReportTotals('requestsCompleted')
+
+    const totalErrors = Object.keys(totals.errors).reduce((total, message) => total + totals.errors[message], 0)
+
+    const errorBudget = analysis.getErrorBudget(script, defaultErrorBudget)
+
+    const finalReport = {
+      errors: totalErrors,
+      reports,
+      totals,
+    }
+
+    if (totalErrors > errorBudget) {
+      finalReport.errorMessage = `${
+        modes.getModeForDisplay(script)
+      } failure: scenarios run: ${
+        totals.scenariosCreated
+      }, total errors: ${
+        totalErrors
+      }, error budget: ${
+        errorBudget
+      }`
+    }
+
+    return finalReport
+  },
+
+  /**
+   * Analyze a set of reports, each of which is the result of an acceptance test's execution
+   * @param timeNow Origin time of the artillery test
+   * @param script Artillery script used
+   * @param settings Platform settings used
+   * @param results Results of the tests
+   *
+   * @returns Object
+   */
+  analyzeAcceptance: (timeNow, script, settings, results) =>
+    analysis.analyzeSamples(script, results, sampling.defaults.acceptance.errorBudget),
+
+  /**
+   * Analyze a set of reports, each of which is the result of an monitoring test's execution
+   * @param timeNow Origin time of the artillery test
+   * @param script Artillery script used
+   * @param settings Platform settings used
+   * @param results Results of the tests
+   *
+   * @returns Object
+   */
+  analyzeMonitoring: (timeNow, script, settings, results) =>
+    analysis.analyzeSamples(script, results, sampling.defaults.monitoring.errorBudget),
+
+  /**
+   * Analyze the performance results.  If there is one payload, then it is a report.  Otherwise, it is a set of payloads.
+   * @param timeNow Origin time of the artillery test
+   * @param script Artillery script used
+   * @param settings Platform settings used
+   * @param results Results of the tests
+   *
+   * @returns Object
+   */
+  analyzePerformance: (timeNow, script, settings, results) => {
+    if (results && results.length === 1) {
+      return results[0] // return the report
+    } else if (results) {
+      return results
+    } else { // eslint-disable-next-line no-underscore-dangle
+      return { message: `load test from ${script._genesis} successfully completed from ${timeNow} @ ${Date.now()}` }
+    }
+  },
+}
+
+module.exports = analysis

test/artillery/analysis.js

@@ -0,0 +1,17 @@
+const sampling = require('./sampling.js')
+const planning = require('./planning.js')
+const analysis = require('./analysis.js')
+
+const artilleryAcceptance = artilleryTask => ({
+  execute: (timeNow, script, settings) => {
+    const acceptanceScript = sampling.applyAcceptanceSamplingToScript(script, settings)
+
+    const plans = planning.planSamples(timeNow, acceptanceScript, settings)
+
+    return artilleryTask
+      .executeAll(acceptanceScript, settings, plans, timeNow)
+      .then(results => analysis.analyzeAcceptance(timeNow, acceptanceScript, settings, results))
+  },
+})
+
+module.exports = artilleryAcceptance

test/artillery/artillery-acceptance.js

@@ -0,0 +1,25 @@
+const sampling = require('./sampling.js')
+const planning = require('./planning.js')
+const analysis = require('./analysis.js')
+const alert = require('./alert.js')
+
+const artilleryMonitoring = artillery => ({
+  execute: (timeNow, script, settings) => {
+    const monitorScript = sampling.applyMonitoringSamplingToScript(script, settings)
+
+    const plans = planning.planSamples(timeNow, monitorScript, settings)
+
+    return artillery.executeAll(monitorScript, settings, plans, timeNow)
+      .then(results => analysis.analyzeMonitoring(timeNow, script, settings, results))
+      .then((monitoringResults) => {
+        if (monitoringResults.errorMessage) {
+          return alert.send(script, monitoringResults)
+            .then(() => monitoringResults)
+        }
+
+        return monitoringResults
+      })
+  },
+})
+
+module.exports = artilleryMonitoring

test/artillery/artillery-monitoring.js

@@ -0,0 +1,13 @@
+const planning = require('./planning.js')
+const analysis = require('./analysis.js')
+
+const artilleryPerformance = artilleryTask => ({
+  execute: (timeNow, script, settings) => {
+    const plans = planning.planPerformance(timeNow, script, settings)
+
+    return artilleryTask.executeAll(script, settings, plans, timeNow)
+      .then(results => analysis.analyzePerformance(timeNow, script, settings, results))
+  },
+})
+
+module.exports = artilleryPerformance