Global jmeter test

jmeter/loadLocalDeviceProperties.py

@@ -0,0 +1,67 @@
+import os
+import json
+
+"""
+Adds local properties to jmeter runtime. 
+Props added to user.properties can be referenced directly by Jmeter script
+with no need of a separate CSV File Reader, making the test localization seamless.
+
+-- device location properties:
+   - country name
+   - country-code
+   - area
+   - locality
+   - ...
+
+-- Region-specific properties: 
+   When adding a .properties file prefixed with the area, i.e {area}.properties the file would be appended to the devices only matching
+   the area in the filename:
+   Example :
+       usa.properties would be appended only to the devices located in US.
+       india.properties would be available only on devices in India
+   Useful for a data-driven test scripts where users from different regions would hit different hosts
+
+"""
+
+jmeter_path = "/opt/apache-jmeter/bin/"
+worker_dir_path = "jmeterWorker"
+device_location_file = "'~/.neocortix/device-location.json'"
+
+
+def get_device_info():
+    device_info = {}
+    with open(os.path.expanduser(device_location_file), "rt") as f:
+        try:
+            device_info = json.load(f)
+        except Exception:
+            print(f"Failed to load file {device_location_file} as JSON")
+            print("Missing localization properties!")
+    return device_info
+
+
+def add_props_to_jmeter_properties(local_props):
+    # appends the properties to user.properties
+    with open(os.path.join(jmeter_path, "user.properties"), "a") as outfile:
+        outfile.write("\n")
+
+        for key, value in local_props.items():
+            outfile.write(f"{key}={value}\n")
+
+        # appending all the local properties, matching the device location
+        # for more strict requirements, locality, country-code can be used, at defined hierarchy.
+        for area_range in ["country", "country-code", "area", "locality"]:
+            try:
+                local_properties_file = os.path.join(worker_dir_path, f"{local_props[area_range]}.properties")
+            except KeyError:
+                print(f"{area_range} data not available on the device!")
+                continue
+            if os.path.isfile(local_properties_file):
+                with open(local_properties_file, "r") as infile:
+                    for line in infile.readlines():
+                        outfile.write(line + "\n")
+
+
+if __name__ == "__main__":
+    localization_data = get_device_info()
+    localization_data["country"] = os.environ["CURRENT_LOCATION"]
+    add_props_to_jmeter_properties(localization_data)

jmeter/localizedJmeterProcessor.py

@@ -0,0 +1,76 @@
+import ncscli.batchRunner as batchRunner
+import glob
+import os
+
+
+class LocalJMeterFrameProcessor(batchRunner.frameProcessor):
+    '''
+    Defines installation and execution of a jmeter test, for a single region within a global context
+
+
+    '''
+
+    def __init__(self, current_instance_count, current_location, number_of_local_instances, number_of_global_instances, test_properties):
+        self.current_location = current_location
+        self.instance_begin_count = current_instance_count
+        self.local_instances = number_of_local_instances
+        self.number_of_global_instances = number_of_global_instances
+        self.test_properties = test_properties
+
+    homeDirPath = '/root'
+    workerDirPath = 'jmeterWorker'
+    JMeterFilePath = workerDirPath+'/XXX.jmx'
+    JVM_ARGS ='-Xms30m -XX:MaxMetaspaceSize=64m -Dnashorn.args=--no-deprecation-warning'
+    # a shell command that uses python psutil to get a recommended java heap size
+    # computes available ram minus some number for safety, but not less than some minimum
+    clause = "python3 -c 'import psutil; print( max( 32000000, psutil.virtual_memory().available-400000000 ) )'"
+
+    def installerCmd( self ):
+        cmd = 'free --mega -t 1>&2'
+        cmd += f" && {self._get_copy_jars_cmd()}"
+        cmd += " && JVM_ARGS='%s -Xmx$(%s)' /opt/apache-jmeter/bin/jmeter.sh --version" % (self.JVM_ARGS, self.clause)
+
+        # tougher pretest
+        pretestFilePath = self.workerDirPath+'/pretest.jmx'
+        if os.path.isfile( pretestFilePath ):
+            cmd += " && cd %s && JVM_ARGS='%s -Xmx$(%s)' /opt/apache-jmeter/bin/jmeter -n -t %s/%s/pretest.jmx -l jmeterOut/pretest_results.csv -D httpclient4.time_to_live=1 -D httpclient.reset_state_on_thread_group_iteration=true" % (
+                self.workerDirPath, self.JVM_ARGS, self.clause, self.homeDirPath, self.workerDirPath
+            )
+        return cmd
+
+    def frameOutFileName( self, frameNum ):
+        return 'jmeterOut_%03d' % frameNum + self.instance_begin_count
+        #return 'TestPlan_results_%03d.csv' % frameNum
+
+    def frameCmd( self, frameNum ):
+        cmd = f"{self._get_id_config_command(frameNum)} && "
+        cmd += f"{self._get_split_files_cmd()} && "
+        cmd += f"""cd {self.workerDirPath} && mkdir -p jmeterOut && JVM_ARGS="{self.JVM_ARGS} -Xmx$({self.clause})" /opt/apache-jmeter/bin/jmeter.sh -n -t {self.homeDirPath}/{self.workerDirPath}/{self.JMeterFilePath} -l jmeterOut/TestPlan_results.csv -D httpclient4.time_to_live=1 -D httpclient.reset_state_on_thread_group_iteration=true"""
+        cmd += f" && mv jmeterOut ~/{self.frameOutFileName(frameNum)}"
+        return cmd
+
+    def _get_copy_jars_cmd(self):
+        cmd = ""
+        if glob.glob(os.path.join( self.workerDirPath, '*.jar')):
+            cmd += ' && cp -p %s/*.jar /opt/apache-jmeter/lib/ext' % self.workerDirPath
+        return cmd
+
+    def _get_update_properties_cmd(self):
+        cmd = f"{self._get_update_localized_properties_cmd()}"
+        return cmd
+
+    def _get_update_localization_properties_cmd(self):
+        cmd = f"python3 {self.homeDirPath}/{self.workerDirPath}/loadLocalDeviceProperties.py"
+        return cmd
+
+    def _get_id_config_command(self, ordered_instance_id):
+        cmd = f"""export GLOBAL_INSTANCE_ID={ordered_instance_id + self.instance_begin_count} && \
+            export LOCAL_INSTANCE_ID={ordered_instance_id} && \
+            export GLOBAL_INSTANCE_COUNT={self.number_of_global_instances} && \
+            export LOCAL_INSTANCE_COUNT={self.local_instances} && \
+            export CURRENT_LOCATION={self.current_location}"""
+        return cmd
+
+    def _get_split_files_cmd(self):
+        cmd = f"python3 {self.homeDirPath}/{self.workerDirPath}/splitFiles.py"
+        return cmd

jmeter/runGlobalJmeter.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+'''Loads configuration from test_plan.json and executes the Jmeter script'''
+import argparse
+import datetime
+import glob
+import logging
+import math
+import os
+import shutil
+import subprocess
+import sys
+import json
+import ncscli.batchRunner as batchRunner
+from localizedJmeterProcessor import LocalJMeterFrameProcessor as JMeterFrameProcessor
+
+logger = logging.getLogger(__name__)
+logFmt = '%(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s'
+logDateFmt = '%Y/%m/%d %H:%M:%S'
+formatter = logging.Formatter(fmt=logFmt, datefmt=logDateFmt )
+logging.basicConfig(format=logFmt, datefmt=logDateFmt)
+logger.setLevel(logging.INFO)
+
+test_plan = {}
+
+
+def scriptDirPath():
+    '''returns the absolute path to the directory containing this script'''
+    return os.path.dirname(os.path.realpath(__file__))
+
+
+try:
+    with open("test_plan.json", "r") as test_plan_file:
+        test_plan = json.load(test_plan_file)
+except Exception as e:
+    logger.error(f"Failed to load test plan file! exception : {e}")
+    sys.exit(1)
+
+
+ap = argparse.ArgumentParser( description=__doc__,
+    fromfile_prefix_chars='@', formatter_class=argparse.ArgumentDefaultsHelpFormatter )
+ap.add_argument( '--authToken', help='the NCS authorization token to use (or none, to use NCS_AUTH_TOKEN env var' )
+ap.add_argument( '--outDataDir', required=True, help='a path to the output data dir for this run (required)' )
+ap.add_argument( '--jtlFile', help='the file name of the jtl file produced by the test plan (if any)',
+    default='TestPlan_results.csv')
+ap.add_argument( '--workerDir', help='the directory to upload to workers',
+    default='jmeterWorker'
+    )
+# for analysis and plotting
+ap.add_argument( '--rampStepDuration', type=float, default=60, help='duration of ramp step, in seconds' )
+ap.add_argument( '--SLODuration', type=float, default=240, help='SLO duration, in seconds' )
+ap.add_argument( '--SLOResponseTimeMax', type=float, default=2.5, help='SLO RT threshold, in seconds' )
+# environmental
+ap.add_argument( '--jmeterBinPath', help='path to the local jmeter.sh for generating html report' )
+ap.add_argument( '--cookie' )
+args = ap.parse_args()
+
+
+workerDirPath = args.workerDir.rstrip( '/' )  # trailing slash could cause problems with rsync
+if workerDirPath:
+    if not os.path.isdir( workerDirPath ):
+        logger.error( 'the workerDirPath "%s" is not a directory', workerDirPath )
+        sys.exit( 1 )
+    JMeterFrameProcessor.workerDirPath = workerDirPath
+else:
+    logger.error( 'this version requires a workerDirPath' )
+    sys.exit( 1 )
+logger.debug( 'workerDirPath: %s', workerDirPath )
+
+jmxFilePath = test_plan["testFile"]
+jmxFullerPath = os.path.join( workerDirPath, jmxFilePath )
+if not os.path.isfile( jmxFullerPath ):
+    logger.error( 'the jmx file "%s" was not found in %s', jmxFilePath, workerDirPath )
+    sys.exit( 1 )
+logger.debug( 'using test plan "%s"', jmxFilePath )
+
+
+jmeterBinPath = args.jmeterBinPath
+if not jmeterBinPath:
+    jmeterVersion = '5.4.1'  # 5.3 and 5.4.1 have been tested, others may work as well
+    jmeterBinPath = scriptDirPath()+'/apache-jmeter-%s/bin/jmeter.sh' % jmeterVersion
+
+
+# use given planDuration unless it is not positive, in which case extract from the jmx
+planDuration = test_plan["testDuration"]
+frameTimeLimit = max( round( planDuration * 1.5 ), planDuration+8*60 ) # some slop beyond the planned duration
+
+JMeterFrameProcessor.JMeterFilePath = jmxFilePath
+
+
+device_requirements = test_plan["device_requirements"]
+device_count = test_plan["device_count"]
+
+total_devices_required = 0
+
+
+for device_prop in device_count:
+    total_devices_required += device_prop["count"]
+
+""""
+
+
+HERE SHOULD BE THE CODE TO EXECUTE BATCHES IN PARALLEL
+
+
+"""
+
+def executeBatch(frameProcessor, filters, timeLimit, nWorkers):
+    try:
+        rc = batchRunner.runBatch(
+            frameProcessor = frameProcessor,
+            commonInFilePath = frameProcessor.workerDirPath,
+            authToken = args.authToken or os.getenv( 'NCS_AUTH_TOKEN' ) or 'YourAuthTokenHere',
+            cookie = args.cookie,
+            encryptFiles=False,
+            timeLimit = timeLimit + 40*60,
+            instTimeLimit = 6*60,
+            frameTimeLimit = frameTimeLimit,
+            filter = filters,
+            outDataDir = outDataDir,
+            startFrame = 1,
+            endFrame = nWorkers,
+            nWorkers = nWorkers,
+            limitOneFramePerWorker = True,
+            autoscaleMax = 1
+        )
+    except KeyboardInterrupt:
+        print("Interruption occured")
+    return rc
+
+nFrames = args.nWorkers
+#nWorkers = round( nFrames * 1.5 )  # old formula
+nWorkers = math.ceil(nFrames*1.5) if nFrames <=10 else round( max( nFrames*1.12, nFrames + 5 * math.log10( nFrames ) ) )
+
+dateTimeTag = datetime.datetime.now().strftime( '%Y-%m-%d_%H%M%S' )
+outDataDir = args.outDataDir
+
+try:
+    if (rc == 0) and os.path.isfile( outDataDir +'/recruitLaunched.json' ):
+        rampStepDuration = args.rampStepDuration
+        SLODuration = args.SLODuration
+        SLOResponseTimeMax = args.SLOResponseTimeMax
+
+        rc2 = subprocess.call( [sys.executable, scriptDirPath()+'/plotJMeterOutput.py',
+            '--dataDirPath', outDataDir,
+            '--rampStepDuration', str(rampStepDuration), '--SLODuration', str(SLODuration),
+            '--SLOResponseTimeMax', str(SLOResponseTimeMax)
+            ],
+            stdout=subprocess.DEVNULL )
+        if rc2:
+            logger.warning( 'plotJMeterOutput exited with returnCode %d', rc2 )
+
+        jtlFileName = args.jtlFile  # make this match output file name from the .jmx (or empty if none)
+        if jtlFileName:
+            nameParts = os.path.splitext(jtlFileName)
+            mergedJtlFileName = nameParts[0]+'_merged_' + dateTimeTag + nameParts[1]
+            rc2 = subprocess.call( [sys.executable, scriptDirPath()+'/mergeBatchOutput.py',
+                '--dataDirPath', outDataDir,
+                '--csvPat', 'jmeterOut_%%03d/%s' % jtlFileName,
+                '--mergedCsv', mergedJtlFileName
+                ], stdout=subprocess.DEVNULL
+                )
+            if rc2:
+                logger.warning( 'mergeBatchOutput.py exited with returnCode %d', rc2 )
+            else:
+                if not os.path.isfile( jmeterBinPath ):
+                    logger.info( 'no jmeter installed for producing reports (%s)', jmeterBinPath )
+                else:
+                    rcx = subprocess.call( [jmeterBinPath,
+                        '-g', os.path.join( outDataDir, mergedJtlFileName ),
+                        '-o', os.path.join( outDataDir, 'htmlReport' )
+                        ], stderr=subprocess.DEVNULL
+                    )
+                    try:
+                        shutil.move( 'jmeter.log', os.path.join( outDataDir, 'genHtml.log') )
+                    except Exception as exc:
+                        logger.warning( 'could not move the jmeter.log file (%s) %s', type(exc), exc )
+                    if rcx:
+                        logger.warning( 'jmeter reporting exited with returnCode %d', rcx )
+    sys.exit( rc )
+except KeyboardInterrupt:
+    logger.warning( 'an interuption occurred')