diff --git a/CSSAnalyzer.py b/CSSAnalyzer.py new file mode 100644 index 0000000..478cad4 --- /dev/null +++ b/CSSAnalyzer.py @@ -0,0 +1,607 @@ + +import os +import json +from bs4 import BeautifulSoup +import traceback +import logging + +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 +SELECTOR_BUILD_FROM_ATTRIBUTES = 15 + +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 + +SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR ] +SELECTORS_ARRAY_NAMES = ["CLASSIC_SELECTOR", "DYMANIC_SELECTOR", "CUSTOM_CSS_SELECTOR" ] + +logger = logging.getLogger(__name__) + +# Description: +# This method will be called to handle the result of filter (by value, text,etc) operation done +# on the selectors found. +# +# There are 3 options for the result: +# 1) No selectors were found having the value we are expecting. On this case, +# information returned will be the element with the index that was expected. +# +# 2) We found only one selector, we have two options here: +# a) Found the correct selector: Return the original element. +# b) Found the incorrect selector. Return two elements, one with the original index and other with the found index. +# +# 3) We found two or more selectors with the same src. We have two options here: +# a) The correct selector was found. Return the original element. +# b) The correct selector was not found. Return two elements, one with the original index and other with all the indexes found. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +# +def processResults(selector, htmlElements, expectedIndex, expectedValue, elementsFoundWithValue, indexesFound, attribute): + jsonObject = {} + elements = [] + + if(elementsFoundWithValue == 0): + # No selectors were found with the expected value + if(expectedIndex <= len(htmlElements)): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + returnCode = NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE + else: + element = {} + element["index"] = "-1" + element["selector"] = "" + returnCode = STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND + elif(elementsFoundWithValue == 1): + if(expectedIndex in indexesFound): + # The expected selector was found and it is the only selector. + element = {} + element["index"] = expectedIndex + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[expectedIndex].text + else: + element["value"] = htmlElements[expectedIndex][attribute] + returnCode = SELECTOR_FOUND_WITH_CORRECT_INDEX + else: + # The incorrect selector was found and this is the only selector with the expected value + element = {} + element["index"] = indexesFound[elementsFoundWithValue -1] + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[indexesFound[elementsFoundWithValue -1]].text + else: + element["value"] = htmlElements[indexesFound[elementsFoundWithValue -1]][attribute] + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + elif(elementsFoundWithValue > 1): + # Several selectors were found with same value + if(expectedIndex in indexesFound): + # The expected element was found on the selectors + element = {} + element["index"] = expectedIndex + element["selector"] = selector + if(attribute == "text"): + element["value"] = htmlElements[expectedIndex].text + else: + element["value"] = htmlElements[expectedIndex][attribute] + elements.append(element) + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX + else: + # The expected element was NOT found on the selector + element = {} + element["index"] = str(indexesFound) + element["selector"] = selector + if(attribute == "text"): + element["value"] = expectedValue + else: + element["value"] = expectedValue + returnCode = MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX + + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = elementsFoundWithValue + jsonObject["selectors"] = element + jsonObject["rc"] = returnCode + + return jsonObject + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the text value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseTextSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + jsonObject = {} + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + + if(expectedValue == ""): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for htmlElment in htmlElements: + selectorText = htmlElment.text.replace("'", "") + if(selectorText.strip() == expectedValue.strip()): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "text") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the src value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseImageSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + + for selectorImg in htmlElements: + # The element source sometimes comes with a relative path, so it starts with '../'. + # This is to remove the dots and compare against the end of expected value. + if(selectorImg['src'] == expectedValue or expectedValue.endswith(selectorImg['src'][2:None])): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "src") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the href value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the same selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseHypertextSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + jsonObject = {} + indexesFound = [] + filteredIndexes = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] + if hasattr(searchInfo, 'text'): + expectedText = searchInfo["text"] + else: + expectedText = "" + + if(expectedValue == ""): + element = {} + element["index"] = expectedIndex + element["selector"] = selector + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = NO_VALUE_PROVIDED_BY_BE + return jsonObject + + for element in htmlElements: + if(element and element.has_attr('href')): + if(element['href'] == expectedValue): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + # If more than 1 element was found using the same value, lest's filter now by text and update + # the indexesFound with the new indexes (hopefully only one!). + if(numberElementsFoundWithValue > 1 and expectedText != ""): + for i in indexesFound: + if(str(htmlElements[i].string) == expectedText): + filteredIndexes.append(i) + if(len(filteredIndexes) > 0 ): + indexesFound = [] + numberElementsFoundWithValue = len(filteredIndexes) + for index in filteredIndexes: + indexesFound.append(index) + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "href") + +# Description: +# This method will be called when two or more selectors were found with +# the same ntagselector value. This method will use the value attribute +# to filter the selctors and try to find the one that was used by the test. +# +# Parameters: +# selector: selector string +# htmlElements: Array of htmlElements found with the selector. +# searchInfo: Object containing infromation related to the DOM analysis (value to find, element type, etc.). +# expectedIndex: The index that is expected to contain the expected value. +# +# Returns: +# jsonObject with the number of selectors found, the selctors and the return code. +def parseValueSelector(selector, htmlElements, searchInfo, expectedIndex, tag): + indexesFound = [] + index = 0 + numberElementsFoundWithValue = 0 + expectedValue = searchInfo["value"] if ('value' in searchInfo) else "" + expectedText = searchInfo["text"] if ('text' in searchInfo) else "" + + for element in htmlElements: + if(('value' in element) and element['value'] == expectedValue ): + numberElementsFoundWithValue += 1 + indexesFound.append(index) + index+=1 + + # If we have text information available and this is a select element, let's try to + # find the correct value using the text + if(numberElementsFoundWithValue == 0 and expectedText != "" and tag == "select"): + return handleSelectElement(selector, htmlElements, expectedText, expectedIndex, indexesFound, tag) + + return processResults(selector, htmlElements, expectedIndex, expectedValue, numberElementsFoundWithValue, indexesFound, "value") + +def handleSelectElement(selector, htmlElements, expectedText, expectedIndex, selectorIndexes, tag): + jsonObject = {} + value = "" + # Let's first verify the selector with the expected index + selectElement = htmlElements[expectedIndex] + options = selectElement.find_all("option") + for option in options: + if(option.text.strip() == expectedText.strip()): + value = option.get("value") + break + + element = {} + element["index"] = expectedIndex + element["value"] = value + element["selector"] = selector + + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECT_ELEMENT_INCORRECT_VALUE + + return jsonObject + + +# Description: +# This function will be called when the selector string returns 0 elements. It means the selector that we have is not working and we need to +# build a new one. This function will will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def buildSelectorFromAttributes(tag, attributes, soup): + jsonObject = {} + + # Start building the CSS selector + selector = tag + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(soup.select(tag + "[id='"+attributes["id"]+"']")) > 0): + selector = selector + "[id='"+attributes["id"]+"']" + if(attributes["name"] != "undef"): + if(len(soup.select(tag + "[name='"+attributes["name"]+"']")) > 0): + selector = selector + "[name='"+attributes["name"]+"']" + if(attributes["type"] != "undef"): + if(len(soup.select(tag + "[type='"+attributes["type"]+"']")) > 0): + selector = selector + "[type='"+attributes["type"]+"']" + + selectorsFound = soup.select(selector) + numberSelectorsFound = len(selectorsFound) + index = 0 + selectorsWithTextIndex = 0 + + bFoundWithText = False + if(numberSelectorsFound > 1 and attributes["text"] != "undef"): + for sel in selectorsFound: + if(sel.string == attributes['text']): + selectorsWithTextIndex = index + bFoundWithText = True + break + index += 1 + + if(numberSelectorsFound == 0 ): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + elif(numberSelectorsFound == 1 or bFoundWithText ): + element = {} + element["selector"] = selector + element["index"] = selectorsWithTextIndex + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + + return jsonObject + + +# Description: +# This function will be called when we have more than one element as a result of the selector query and we DO NOT have +# a value to try to filter the attributes and find the correct one. So, using the object's attributes we will try to find the +# correct selector. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def findIndexFromAttributes(selector, tag, attributes, soup): + jsonObject = {} + idFound = False + if(attributes["id"] != "false" and attributes["id"] != "undef"): + objectID = attributes["id"] + if(attributes["name"] != "undef"): + objectName = attributes["name"] + if(attributes["type"] != "undef"): + objectType = attributes["type"] + + htmlElementsFound = soup.select(selector) + numberElementsFound = len(htmlElementsFound) + if(numberElementsFound > 1 ): + newIndex = 0 + for element in htmlElementsFound: + if(element.has_attr('id') and element['id'] == objectID and + element.has_attr('name') and element['name'] == objectName and + element.has_attr('type') and element['type'] == objectType): + idFound = True + break + newIndex = newIndex + 1 + + if(idFound): + element = {} + element["selector"] = selector + element["index"] = newIndex + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + else: + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + return jsonObject + +# Description: +# This function will be called when the object DOES NOT HAVE a selector string, so we need to build it from scratch. +# This function will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# soup: (BeautifulSoup object to query HTML DOM) +# Returns: +# CSS selector +# +def buildCSSSelector(tag, attributes, searchInfo, index, soup): + jsonObject = {} + cssSelector = tag + searchType = searchInfo["searchType"] + value = searchInfo["value"] + + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(soup.select(tag + "[id='"+attributes["id"]+"']")) > 0): + cssSelector = cssSelector + "[id = '" + attributes["id"] + "']" + if(attributes["name"] != "undef"): + if(len(soup.select(tag + "[name='"+attributes["name"]+"']")) > 0): + cssSelector = cssSelector + "[name = '" + attributes["name"] + "']" + if(attributes["type"] != "undef"): + if(len(soup.select(tag + "[type='"+attributes["type"]+"']")) > 0): + cssSelector = cssSelector + "[type = '" + attributes["type"] + "']" + + # now that we have a selector, let's va;idate it returns elements. + htmlElementsFound = soup.select(cssSelector) + numberElementsFound = len(htmlElementsFound) + logger.info("buildCSSSelector - Found " + str(numberElementsFound) + " with selector " + cssSelector) + + if(numberElementsFound == 1): + element = {} + element["selector"] = cssSelector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + elif(numberElementsFound > 1): + if(searchType == "value"): + jsonObject = parseValueSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(cssSelector, htmlElementsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + element = {} + element["selector"] = cssSelector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + element = {} + element["selector"] = "" + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + + return jsonObject + + +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def obtainCSSFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + logging.info("Starting CSS analysis for "+ SELECTORS_ARRAY_NAMES[selector_type] + " selector " + str(selector) + " witn index " + str(index) + " on step " + str(stepId)) + jsonObject = {} + path = 'build/reports/geb/' + browserName + '/' + filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): + try: + searchType = searchInfo["searchType"] + value = searchInfo["value"] + text = open(filename, 'r').read() + soup = BeautifulSoup(text, 'html.parser') + if(selector is None): + if(selector_type == CUSTOM_CSS_SELECTOR): + logging.info("NO CSS Selector, need to build it") + return buildCSSSelector(tag, attributes, searchInfo, 0, soup) + else: + selectorsFound = soup.select(selector) + numberSelectorsFound = len(selectorsFound) + + if(action == "mouseover"): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + if(numberSelectorsFound == 0): + if(len(attributes) > 0 and selector_type == CUSTOM_CSS_SELECTOR ): + return buildSelectorFromAttributes(tag, attributes, soup) + else: + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + + elif(numberSelectorsFound == 1 ): + element = {} + element["selector"] = selector + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + if(index > 0): + element["index"] = 0 + if(searchType == "value" and selectorsFound[0].has_attr('value')): + element["value"] = selectorsFound[0]["value"] + elif(searchType == "href" and selectorsFound[0].has_attr('href')): + element["value"] = selectorsFound[0]["href"] + elif(searchType == "text" and selectorsFound[0].has_attr('text')): + element["value"] = selectorsFound[0].text + elif(searchType == "imgsrc" and selectorsFound[0].has_attr('src')): + element["value"] = selectorsFound[0]["src"] + else: + element["value"] = searchInfo["value"] + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + else: + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + elif(numberSelectorsFound > 1 ): + if(value != "undef"): + if(searchType == "value"): + jsonObject = parseValueSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "href"): + jsonObject = parseHypertextSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "text"): + jsonObject = parseTextSelector(selector, selectorsFound, searchInfo, index, tag) + elif(searchType == "imgsrc"): + jsonObject = parseImageSelector(selector, selectorsFound, searchInfo, index, tag) + else: + # Backend sent an undef searchType, we will return no info + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + # We do not have a value to try to identify the element, let's use the attributes + if(len(attributes) > 0 and selector_type == CUSTOM_CSS_SELECTOR ): + logging.info("Found "+ str(numberSelectorsFound) + " selectors and no value to filter, let's build from attributes") + return findIndexFromAttributes(selector, tag, attributes, soup) + else: + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = NO_SEARCH_TYPE_PROVIDED_BY_BE + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + + jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound + + except Exception as ex: + logging.error (ex) + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + logging.error("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) + jsonObject = {} + + return jsonObject diff --git a/XPATHAnalyzer.py b/XPATHAnalyzer.py new file mode 100644 index 0000000..b85e86e --- /dev/null +++ b/XPATHAnalyzer.py @@ -0,0 +1,255 @@ + +from logging import root +import os +import json +import pprint +import lxml.etree +import lxml.html +from xml.sax import saxutils +from lxml import html +import traceback +import logging + +UNKNOWN_ERROR = 1 +NO_TAG_PROVIDED_BY_BE = 2 +NO_VALUE_PROVIDED_BY_BE = 3 +NO_SEARCH_TYPE_PROVIDED_BY_BE = 4 +ACTION_NOT_VALID_FOR_ANALYSIS = 5 +STEP_INDEX_GREATER_THAN_NUMBER_OF_SELECTORS_FOUND = 6 +ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR = 7 +NO_SELECTOR_FOUND_WITH_SPECIFIC_VALUE = 8 +SELECTOR_FOUND_WITH_CORRECT_INDEX = 9 +SELECTOR_FOUND_WITH_INCORRECT_INDEX = 10 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_CORRECT_INDEX = 11 +MULTIPLE_SELECTORS_FOUND_WITH_EXPECTED_VALUE_INCORRECT_INDEX = 12 +NO_SELECTOR_FOUND_WITH_NTAGSELECTOR = 13 +SELECT_ELEMENT_INCORRECT_VALUE = 14 +SELECTOR_BUILD_FROM_ATTRIBUTES = 15 + +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 + +logger = logging.getLogger(__name__) + +# Description: +# This method returns the inner HTML of the element received as a parameter. +# +# Parameters: +# element: HTML element +# +# Returns: +# inner HTML (string) +# +def inner_html(element): + return (saxutils.escape(element.text) if element.text else '') + \ + ''.join([html.tostring(child, encoding=str) for child in element.iterchildren()]) + +# Description: +# This function build an XPATH selector string. It will verify the object's attributes and validate an HTML element +# exists using this attribute before adding it to the selector string. +# +# Parameters: +# tag: HTML tag of the element +# attributes: Object's attributes (name, id, type, etc) +# root: (LXML object to query HTML DOM) +# Returns: +# XPATH selector +# +def buildXPATHSelector(tag, attributes, root): + logger.info("buildXPATHSelector - attributes = " + str(attributes)) + jsonObject = {} + elements = [] + textUsedOnSelector = False + + selector = "//" + tag + if(attributes["id"] != "false" and attributes["id"] != "undef"): + if(len(root.xpath("//" + tag + "[@id='"+attributes["id"]+"']")) > 0): + selector = selector + "[@id='"+attributes["id"]+"']" + + if(attributes["name"] != "undef"): + if(len(root.xpath("//" + tag + "[@name='"+attributes["name"]+"']")) > 0): + selector = selector + "[@name='"+attributes["name"]+"']" + + if(attributes["type"] != "undef"): + if(len(root.xpath("//" + tag + "[@type='"+attributes["type"]+"']")) > 0): + selector = selector + "[@type='"+attributes["type"]+"']" + + if( attributes["text"] and len(attributes["text"]) > 0 and attributes["text"] != "undef" ): + textUsedOnSelector = True + text = attributes["text"] + innerHTMLText = "" + + # As MuukTest sends the text that is obtained from innerText it does not contain any tag that the DOM will contian when searching + # for a XPATH expression. So, we will query all the tag elements and get the innerText so we can compare and then build the xpath + elements = root.xpath("//" + tag) + elementIndex = -1 + for element in elements: + if(element.text_content() == text): + elementIndex = elementIndex + 1 + text = element.text_content() + innerHTMLText = inner_html(element) + + splittedText = text.split("\n") + if(len(splittedText) > 1 or (innerHTMLText != "" and innerHTMLText != text ) ): + # if we are using normalize-space we need to use the complete text but we need to escape invalid characters. + + text = text.replace("\n", " ") + text = text.replace("'", "\'") + text = text.replace("$", '\\$') + text = text.strip() + + selector = selector + "[normalize-space() = \""+text+"\"]" + try: + htmlElementsFound = root.xpath(selector) + except Exception as ex: + # if we failed to obtain selectors, lets use only the tag and the index + selector = "//" + tag + element = {} + element["selector"] = selector + element["index"] = elementIndex + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + return jsonObject + + + else: + if(len(attributes["text"]) > 40): + text = attributes["text"][0:40] + + # Some characters will cause problems on the XPATH expression when using contains, we need to escape the next + # characters: + #logging.info(repr(text)) + #text = text.replace("$", '\$') // no need to escape + text = text.replace("'", "\'") + text = text.strip() + + selector = selector + "[contains(text(),\""+text+"\")]" + + try: + htmlElementsFound = root.xpath(selector) + except Exception as ex: + selector = "//" + tag + element = {} + element["selector"] = selector + element["index"] = elementIndex + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + return jsonObject + + logger.info("buildXPATHSelector - Selector build from attributes = " + str(selector)) + htmlElementsFound = root.xpath(selector) + numberhtmlElementsFound = len(htmlElementsFound) + logger.info("buildXPATHSelector - numberhtmlElementsFound = " + str(numberhtmlElementsFound)) + + if(numberhtmlElementsFound == 0 ): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelector"] = 0 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + jsonObject["rc"] = NO_SELECTOR_FOUND_WITH_NTAGSELECTOR + elif(numberhtmlElementsFound == 1 ): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = 1 + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 1 + elif(numberhtmlElementsFound > 1 or textUsedOnSelector): + element = {} + element["selector"] = selector + element["index"] = 0 + jsonObject["selectors"] = element + jsonObject["rc"] = SELECTOR_BUILD_FROM_ATTRIBUTES + jsonObject["numberOfElementsFoundWithSelector"] = numberhtmlElementsFound + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberhtmlElementsFound + + + return jsonObject + +# Description: +# This method will be call for each step and will parse the DOM files generated +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def obtainXPATHFeedbackFromDOM(classname, stepId, selector, index, tag, type, action, searchInfo, browserName, attributes, selector_type): + logging.info("Starting XPATH analysis for selector " + str(selector) + " witn index " + str(index) + " on step " + str(stepId)) + jsonObject = {} + path = 'build/reports/geb/' + browserName + '/' + filename = path + classname + "_" + str(stepId) + ".html" + if os.path.exists(filename): + try: + searchType = searchInfo["searchType"] + value = searchInfo["value"] + text = open(filename, 'r').read() + root = lxml.html.fromstring(text) + if(selector is None): + logging.info("No XPATH selector found, let's build from attributes") + return buildXPATHSelector(tag, attributes, root) + else: + htmlElementsFound = root.xpath(selector) + numberSelectorsFound = len(htmlElementsFound) + + if(action == "mouseover"): + element = {} + element["selector"] = selector + element["index"] = index + jsonObject["selectors"] = element + jsonObject["rc"] = ACTION_NOT_VALID_FOR_ANALYSIS + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = 0 + else: + if(numberSelectorsFound == 0): + logging.info("Found "+ str(numberSelectorsFound) + " selectors and no value to filter, let's build from attributes") + return buildXPATHSelector(tag, attributes, root) + + elif(numberSelectorsFound == 1 ): + element = {} + element["selector"] = selector + if(index > 0): + element["index"] = 0 + returnCode = SELECTOR_FOUND_WITH_INCORRECT_INDEX + else: + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + + elif(numberSelectorsFound > 1 ): + element = {} + element["selector"] = selector + element["index"] = index + returnCode = ONE_SELECTOR_FOUND_FOR_NTAGSELECTOR + + jsonObject["selectors"] = element + jsonObject["numberOfElementsFoundWithSelectorAndValue"] = numberSelectorsFound + jsonObject["rc"] = returnCode + + jsonObject["numberOfElementsFoundWithSelector"] = numberSelectorsFound + + except Exception as ex: + logging.error(ex) + + # Let's validate the data we generated is a valid json for this step + try: + json.loads(json.dumps(jsonObject)) + except Exception as ex: + logging.error("Invalid JSON format for step " + str(stepId) +" found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) + jsonObject = {} + + return jsonObject \ No newline at end of file diff --git a/appspec.yml b/appspec.yml new file mode 100644 index 0000000..79f6ba0 --- /dev/null +++ b/appspec.yml @@ -0,0 +1,74 @@ +# This is an appspec.yml file for use with an EC2/On-Premises deployment in CodeDeploy. +# The lines in this file starting with the hashtag symbol are +# instructional comments and can be safely left in the file or +# ignored. +# For help completing this file, see the "AppSpec File Reference" in the +# "CodeDeploy User Guide" at +# https://docs.aws.amazon.com/codedeploy/latest/userguide/app-spec-ref.html +version: 0.0 +# Specify "os: linux" if this revision targets Amazon Linux, +# Red Hat Enterprise Linux (RHEL), or Ubuntu Server +# instances. +# Specify "os: windows" if this revision targets Windows Server instances. +# (You cannot specify both "os: linux" and "os: windows".) +os: linux +# During the Install deployment lifecycle event (which occurs between the +# BeforeInstall and AfterInstall events), copy the specified files +# in "source" starting from the root of the revision's file bundle +# to "destination" on the Amazon EC2 instance. +# Specify multiple "source" and "destination" pairs if you want to copy +# from multiple sources or to multiple destinations. +# If you are not copying any files to the Amazon EC2 instance, then remove the +# "files" section altogether. A blank or incomplete "files" section +# may cause associated deployments to fail. +files: + - source: / + destination: /home/ubuntu/projects/executor/ +hooks: +# For each deployment lifecycle event, specify multiple "location" entries +# if you want to run multiple scripts during that event. +# You can specify "timeout" as the number of seconds to wait until failing the deployment +# if the specified scripts do not run within the specified time limit for the +# specified event. For example, 900 seconds is 15 minutes. If not specified, +# the default is 1800 seconds (30 minutes). +# Note that the maximum amount of time that all scripts must finish executing +# for each individual deployment lifecycle event is 3600 seconds (1 hour). +# Otherwise, the deployment will stop and CodeDeploy will consider the deployment +# to have failed to the Amazon EC2 instance. Make sure that the total number of seconds +# that are specified in "timeout" for all scripts in each individual deployment +# lifecycle event does not exceed a combined 3600 seconds (1 hour). +# For deployments to Amazon Linux, Ubuntu Server, or RHEL instances, +# you can specify "runas" in an event to +# run as the specified user. For more information, see the documentation. +# If you are deploying to Windows Server instances, +# remove "runas" altogether. +# If you do not want to run any commands during a particular deployment +# lifecycle event, remove that event declaration altogether. Blank or +# incomplete event declarations may cause associated deployments to fail. + +# During the BeforeInstall deployment lifecycle event, run the commands +# in the script specified in "location". + BeforeInstall: + #- location: scripts/backup + # timeout: 300 + # runas: root + - location: scripts/install_dependencies + timeout: 300 + runas: root +# During the AfterInstall deployment lifecycle event, run the commands +# in the script specified in "location". + AfterInstall: + - location: scripts/config_files + timeout: 300 + runas: root + - location: scripts/start_services + timeout: 300 + runas: root +# During the ApplicationStop deployment lifecycle event, run the commands +# in the script specified in "location" starting from the root of the +# revision's file bundle. + ApplicationStop: + - location: scripts/stop_services + timeout: 300 + runas: root + diff --git a/build.gradle b/build.gradle index afdc692..9028dfe 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ ext { groovyVersion = '2.4.12' gebVersion = '2.2' seleniumVersion = '3.6.0' - chromeDriverVersion = '79.0.3945.36' + chromeDriverVersion = '97.0.4692.71' geckoDriverVersion = '0.23.0' } } @@ -39,6 +39,9 @@ dependencies { testCompile "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion" compile group: 'org.seleniumhq.selenium', name: 'selenium-support', version: '3.14.0' + + // JSON simple for MuukReport + implementation group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1' } diff --git a/domparser.py b/domparser.py new file mode 100644 index 0000000..6b27208 --- /dev/null +++ b/domparser.py @@ -0,0 +1,185 @@ + +import os +import json +import pprint +from bs4 import BeautifulSoup +from CSSAnalyzer import obtainCSSFeedbackFromDOM +from XPATHAnalyzer import obtainXPATHFeedbackFromDOM +import traceback +import logging +import shutil +import pathlib + + +CLASSIC_SELECTOR = 0 +DYMANIC_SELECTOR = 1 +CUSTOM_CSS_SELECTOR = 2 +XPATH_SELECTOR = 3 + +SELECTORS_ARRAY = [CLASSIC_SELECTOR, DYMANIC_SELECTOR, CUSTOM_CSS_SELECTOR, XPATH_SELECTOR ] +LOG_FILE_NAME = "DOMParser.log" + +logging.basicConfig(filename = LOG_FILE_NAME, + filemode = "w", + format = "%(levelname)s %(asctime)s - %(message)s", + level = logging.INFO) +logger = logging.getLogger(__name__) + + +# Description: +# This method will be call to execuete the Muuk Report analysis. +# for the test to find the selectors for this step. If more than one selector is found, +# this method makes another search on the DOM using the value to filter the +# selectors found. +# +# Returns: +# jsonObject with the number of selector information. +def createMuukReport(classname, browserName): + logging.info("Starting parsing analysis") + path = 'build/reports/' + filename = path + classname + ".json" + muukReport = {} + steps = [] + if(os.path.exists(filename)): + try: + jsonFile = open(filename, 'r') + elements = json.load(jsonFile) + for element in elements['stepsFeedback']: + type = element.get("type") + if(type == "step"): + element["feedback"] = [] + selectors = json.loads(element.get("selectors").replace('\$', '\\\$')) + selectorToUse = element.get("selectorToUse") + + try: + valueInfo = json.loads(element.get("value")) + except Exception as ex: + valueInfo = {"value":"","":"href","":""} + + try: + attributes = json.loads(element.get("attributes")) + except Exception as ex: + attributes = {"id":"false","name":"undef","text":"","type":"undef"} + + attributes['value'] = valueInfo['value'] + + for i in SELECTORS_ARRAY: + + if(i < len(selectors)): + selector = selectors[i]['selector'] + if(selector == ""): + selector = None + index = selectors[i]['index'] + else: + selector = None + index = None + + if(i < 3): + domInfo = obtainCSSFeedbackFromDOM(classname, element.get("id"), + selector, + index, + element.get("tag"), + element.get("objectType"), element.get("action"), + valueInfo, + browserName, + attributes, + SELECTORS_ARRAY[i]) + else: + if(selector != None): + # XPATH library is case sensitive and MuukTest creates the tag as upper case, we need to fix this. + selector = selector.replace(element.get("tag").upper(), element.get("tag"), 1) + domInfo = obtainXPATHFeedbackFromDOM(classname, element.get("id"), + selector, + index, + element.get("tag"), + element.get("objectType"), element.get("action"), + valueInfo, + browserName, + attributes, + SELECTORS_ARRAY[i]) + + + try: + logging.info("Object = " + json.dumps(domInfo,sort_keys=True, indent=4)) + except Exception as ex: + logging.error("Invalid domInfo generated") + + if(domInfo): + element["feedback"].append(domInfo) + steps.append(element) + + # Now that we have the 4 selector arrays, let's define which one we should use + selectorToUse = findBetterSelectorToUse(element["feedback"], attributes) + element["feebackSelectorToUse"] = selectorToUse + + else: + steps.append(element) + + except Exception as ex: + logging.error("Exception found during DOM parsing. Exception = ") + logging.error(ex) + logging.error(traceback.format_exc()) + + # Closing file + jsonFile.close() + else: + logging.error("Muuk Report was not found!") + + # Let's validate the data we generated is a valid json + try: + json.loads(json.dumps(steps)) + muukReport["steps"] = steps + except Exception as ex: + logging.error("Invalid JSON format was found, will not send feedback to BE") + logging.error(ex) + logging.error(traceback.format_exc()) + muukReport["steps"] = {} + + # Print report if touch file exists + if(os.path.exists("TOUCH_TRACE_REPORT")): + pprint.pprint((muukReport["steps"])) + + logging.info("Final Feedback Object:") + logging.info(json.dumps(muukReport["steps"],sort_keys=True, indent=4)) + + # Last step is to copy the log file to build folder + try: + source = str(pathlib.Path(__file__).parent.resolve()) + "/" + LOG_FILE_NAME + destination = str(pathlib.Path(__file__).parent.resolve()) + "/build/" +LOG_FILE_NAME; + shutil.copyfile(source, destination) + except Exception as ex: + print(ex) + + return muukReport + +# if the Object has text, we will use XPATH selector. +# else if Object has the next attributes ['id', 'name', 'type', 'role', 'title'], use custom CSS +# else if ntagselector has dynamic classes, use dynamic selector +# else use clasic selector. +def findBetterSelectorToUse(selectors, attributes): + + selectorToUse = -1 + classic = selectors[0] if len(selectors) > 0 else [] + dynamic = selectors[1] if len(selectors) > 1 else [] + customeCSS = selectors[2] if len(selectors) > 2 else [] + xpath = selectors[3] if len(selectors) > 3 else [] + + if(xpath and xpath["numberOfElementsFoundWithSelectorAndValue"] > 0 and attributes["text"] != "undef" and + ("contains" in xpath["selectors"]["selector"] or "normalize-space" in xpath["selectors"]["selector"] )): + selectorToUse = 3 + elif(customeCSS and (attributes["id"] != "undef" or attributes["name"] != "undef" or attributes["type"] != "undef" ) and customeCSS["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 2 + elif(classic and classic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 0 + elif(dynamic and dynamic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 1 + + # If we were not able to choose a selector with values, check if we have one that return any element at least. + if(selectorToUse == -1): + if(classic and classic["numberOfElementsFoundWithSelector"] > 0): + selectorToUse = 0 + elif(dynamic and dynamic["numberOfElementsFoundWithSelectorAndValue"] > 0): + selectorToUse = 1 + + return selectorToUse + diff --git a/mkcli.py b/mkcli.py index c24a245..8090a24 100644 --- a/mkcli.py +++ b/mkcli.py @@ -11,6 +11,7 @@ from mkcloud import gatherScreenshots, resizeImages from mkvideo import Video #import ssl +from domparser import createMuukReport def gatherFeedbackData(browserName): #The path will be relative to the browser used to execute the test (chromeTest/firefoxTest) @@ -21,6 +22,7 @@ def gatherFeedbackData(browserName): for filename in os.listdir(path): testSuccess = True error = '' + failureMessage = '' if filename.endswith('.xml'): e = xml.etree.ElementTree.parse('build/test-results/'+browserName+'/' + filename).getroot() @@ -31,7 +33,12 @@ def gatherFeedbackData(browserName): if e.find('testcase') is not None : if e.find('testcase').find('failure') is not None : error = e.find('testcase').find('failure').attrib['message'] - + if(error.find("Build info") != -1): + error = error.split(sep = "Build", maxsplit=1)[0] + failureMessage = e.find('testcase').find('failure').text + if(failureMessage.find("Build info") != -1): + failureMessage = failureMessage.split(sep = "Build", maxsplit=1)[0] + testResult = { "className": e.attrib['name'] if e.attrib['name'] is not None else "", "success": testSuccess, @@ -39,12 +46,19 @@ def gatherFeedbackData(browserName): "hostname": e.attrib['hostname'] if e.attrib['hostname'] is not None else "", "executionTime": e.attrib['time'] if e.attrib['time'] is not None else "", "error": error, - "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "" + "systemoutput": e.find('system-out').text if e.find('system-out') is not None else "", + "systemerror": e.find('system-err').text if e.find('system-err') is not None else "", + "failureMessage": failureMessage, + "muukReport": {}, } + # get and attach the report insights found during script execution + testResult["muukReport"] = createMuukReport(testResult.get("className"), browserName) + feedbackData.append(testResult) else: print("gatherFeedbackData - path does not exists ") testResult = { + "muukReport": {}, "success" : False, #"executionAt": "", "error" : "Test failed during execution. This could be compilation error", @@ -63,13 +77,24 @@ def run(args): noexec = args.noexec route = 'src/test/groovy' browser = args.browser + # internal cloud only + executionNumber = args.executionnumber or None + scheduleExecutionNumber = args.scheduleexecutionnumber or None + + if scheduleExecutionNumber is not None: + scheduleExecutionNumber = int(scheduleExecutionNumber) + else: + scheduleExecutionNumber = 0 + + origin = args.origin or None + originid = args.originid or None + ######## dimensions = args.dimensions if dimensions is not None: checkDimensions = isinstance(dimensions[0], int) & isinstance(dimensions[1],int) else: checkDimensions = False - executionNumber = None #Exit code to report at circleci exitCode = 1 #Check if we received a browser and get the string for the gradlew task command @@ -99,7 +124,7 @@ def run(args): token='' try: key_file = open(path,'r') - key = key_file.read() + key = key_file.read().strip() r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}) #r = requests.post(muuktestRoute+"generate_token_executer", data={'key': key}, verify=False) responseObject = json.loads(r.content) @@ -128,10 +153,16 @@ def run(args): print(dest) shutil.copytree(route, dest) shutil.copytree("build/", dest+"/build") + #internal cloud only + files = [f for f in os.listdir(".") if f.endswith('.mp4')] + shutil.move(files[0], dest) if len(files) == 1 else print("Not a video to backup") + ######### + shutil.rmtree(route, ignore_errors=True) os.makedirs(route) - values = {'property': field, 'value[]': valueArr, 'userId': userId} + #values = {'property': field, 'value[]': valueArr, 'userId': userId, 'executionnumber': executionNumber} + values = {'property': field, 'value[]': valueArr, 'userId': userId, 'executionnumber': executionNumber, 'origin': origin, 'originid': originid, 'scheduleExecutionNumber': scheduleExecutionNumber} # Add screen dimension data if it is set as an argument if checkDimensions == True: values['dimensions'] = [dimensions[0],dimensions[1]] @@ -191,7 +222,7 @@ def run(args): try: requests.post(supportRoute+"tracking_data", json=payload) - # equests.post(supportRoute+"tracking_data", json=payload, verify=False) + #requests.post(supportRoute+"tracking_data", json=payload, verify=False) except Exception as e: print("No connection to support Data Base") @@ -199,26 +230,28 @@ def run(args): if noexec == False : #Execute the test videoNameFile = str(organizationId) + "_" + str(executionNumber) + ".mp4" + v = Video() print("File name for video: " + videoNameFile) print("Executing test...") try: - v = Video() v.checkAndStartRecording(videoNameFile) - v.checkActiveSession() - v.executeCmd("ps -ef | grep ffmpeg") - v.executeCmd("ls -ltr | grep *.mp4") + #v.checkActiveSession() + #v.executeCmd("ps -ef | grep ffmpeg") + #v.executeCmd("ls -ltr | grep *.mp4") exitCode = subprocess.call(dirname + '/gradlew clean '+browserName, shell=True) except Exception as e: print("Error during gradlew compilation and/or execution ") print(e) - v.executeCmd("ps -ef | grep ffmpeg") + #v.executeCmd("ps -ef | grep ffmpeg") v.checkAndStopRecording() - v.executeCmd("ls -ltr | grep *.mp4") + #v.executeCmd("ls -ltr | grep *.mp4") del v testsExecuted = gatherFeedbackData(browserName) url = muuktestRoute+'feedback/' - values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber)} + #values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber)} + values = {'tests': testsExecuted, 'userId': userId, 'browser': browserName,'executionNumber': int(executionNumber), 'origin': origin, 'originid': originid, 'scheduleExecutionNumber': scheduleExecutionNumber} + hed = {'Authorization': 'Bearer ' + token} #CLOUD SCREENSHOTS STARTS # @@ -281,6 +314,10 @@ def main(): parser.add_argument("-noexec",help="(Optional). If set then only download the scripts", dest="noexec", action="store_true") parser.add_argument("-browser",help="(Optional). Select one of the available browsers to run the test (default firefox)", type=str, dest="browser") parser.add_argument("-dimensions",help="(Optional). Dimensions to execute the tests, a pair of values for width height, ex. -dimensions 1800 300", type=int, nargs=2, dest="dimensions") + parser.add_argument("-executionnumber",help="(Optional) this numbers contain the executionnumber from the cloud execution", type=str, dest="executionnumber") + parser.add_argument("-origin",help="Test origin, this is cloud only", type=str, dest="origin") + parser.add_argument("-originid",help="Test origin id (scheduling)", type=str, dest="originid") + parser.add_argument("-scheduleexecutionnumber",help="(Optional) this numbers contain the executionnumber (scheduling)", type=str, dest="scheduleexecutionnumber") parser.set_defaults(func=run) args=parser.parse_args() args.func(args) diff --git a/scripts/backup b/scripts/backup new file mode 100644 index 0000000..c5a9c2a --- /dev/null +++ b/scripts/backup @@ -0,0 +1,6 @@ +#!/bin/bash +if [ -d "/home/ubuntu/projects/executor" ]; then + # Control will enter here if $DIRECTORY exists. + mkdir -p /home/ubuntu/backup + cp -r /home/ubuntu/projects/executor /home/ubuntu/backup/executor +fi \ No newline at end of file diff --git a/scripts/config_files b/scripts/config_files new file mode 100755 index 0000000..eb24a4b --- /dev/null +++ b/scripts/config_files @@ -0,0 +1,29 @@ +#!/bin/bash +#Verifying the instance +#Finding the current ip +IP=$(hostname -I | awk '{print $1}') +#Testing Instance +testingInstance="172.31.20.165" +#Staging Instance +stagingInstance="172.31.12.113" +#172.31.1.41 Ari Testing Instance +testingInstanceTmp="172.31.1.41" + +echo $IP +if [[ "$IP" == $testingInstance || "$IP" == $testingInstanceTmp ]]; then + echo "It's the testing instance" + #Change portal route + sudo sed -i 's/portal.muuktest.com/testing.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py +elif [[ "$IP" == $stagingInstance ]]; then + echo "It's the staging instance" + #Change portal route + sudo sed -i 's/portal.muuktest.com/staging.muuktest.com/g' /home/ubuntu/projects/executor/mkcli.py +fi + +#Setting the cloud key +#The cloud key it's located in a external env.json file +if [[ -f "/home/ubuntu/env.json" ]]; then + cloudKey=$(cat /home/ubuntu/env.json | jq -r '.cloudKey') + sudo sed -i '$ a def getCloudKey():' /home/ubuntu/projects/executor/mkcloud.py + sudo sed -i "$ a \ \ return('$cloudKey')" /home/ubuntu/projects/executor/mkcloud.py +fi \ No newline at end of file diff --git a/scripts/install_dependencies b/scripts/install_dependencies new file mode 100644 index 0000000..7e6b874 --- /dev/null +++ b/scripts/install_dependencies @@ -0,0 +1,21 @@ +#!/bin/bash +#Installing Pip3 +sudo apt -y install python3-pip +#Installing BeautifulSoup4 +sudo pip3 install BeautifulSoup4 +#sudo pip3 install bs4 +#Installing xvfb +sudo apt -y install xvfb +#Installing firefox +sudo apt -y install firefox +#Installing google chrome +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +sudo apt -y install ./google-chrome-stable_current_amd64.deb +sudo apt-get install google-chrome-stable +sudo rm google-chrome-stable_current_amd64.deb +#Installing ffmpeg +sudo apt -y install ffmpeg +#Installing request +pip3 install request +#Installing jq to read json files +sudo apt -y install jq \ No newline at end of file diff --git a/scripts/start_services b/scripts/start_services new file mode 100644 index 0000000..e2d58ea --- /dev/null +++ b/scripts/start_services @@ -0,0 +1,10 @@ +#!/bin/bash +sudo chmod 777 /home/ubuntu/projects/executor/gradlew +Xvfb :99 -screen 0 1366x768x16 & + +#Starting queue service in case it is needed +serviceStatus=`systemctl is-active muuktest-queued.service` +queueServiceFile=/lib/systemd/system/muuktest-queued.service +if [[ "$serviceStatus" != 'active' ]] && [ -f "$queueServiceFile" ]; then + sudo systemctl start muuktest-queued.service +fi \ No newline at end of file diff --git a/scripts/stop_services b/scripts/stop_services new file mode 100644 index 0000000..29cb840 --- /dev/null +++ b/scripts/stop_services @@ -0,0 +1,7 @@ +#!/bin/bash +#Stoping queue service in case it is needed +serviceStatus=`systemctl is-active muuktest-queued.service` +if [[ "$serviceStatus" == 'active' ]]; then + echo "muuktest-queued.service is running" + sudo systemctl stop muuktest-queued.service +fi \ No newline at end of file