From aecd269aec4873153ed7b5b8dd4b2f1c69d1a7f0 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Thu, 25 May 2023 12:14:13 +1000 Subject: [PATCH] Fix: pre-commit and tox issues pre-commit and tox issues need to be fixed for tox-verify GHA tox-verify to work correctly. - shellcheck errors: SC3040 (warning): In POSIX sh, set option pipefail is undefined. SC3037 (warning): In POSIX sh, echo flags are undefined. SC2086 (info): Double quote to prevent globbing and word splitting. SC2004 (style): $/${} is unnecessary on arithmetic variables. - flake: E501 line too long - pydocsytle errors: D103: Missing docstring in public function D100: Missing docstring in public module - write-good errors remove passive voice - Fixes bandit: CWE: CWE-20 (https://cwe.mitre.org/data/definitions/20.html) Using lxml.etree.parse to parse untrusted XML data is known to be vulnerable to XML attacks. Replace lxml.etree.parse with its defusedxml equivalent function Ref: https://rules.sonarsource.com/python/RSPEC-2755 Issue: LF-JIRA RELENG-4742 Change-Id: I635281bc39ebb1157df13dde6809c2809af8c1d4 Signed-off-by: Anil Belur --- jjb/docs/docs-post-rtd.sh | 2 +- jjb/integration/common-functions.sh | 2 +- .../integration-get-slave-addresses.sh | 2 +- jjb/integration/integration-macros.yaml | 1 - jjb/lf-infra-releasedockerhub.yaml | 49 +++++----- jjb/opendaylight-infra-deploy-maven-site.sh | 1 + jjb/releng-maven-mri-stage.sh | 1 + jjb/releng-maven-mri-stage.yaml | 1 + jjb/run_releasedockerhub.sh | 1 + scripts/bump_mri_versions/main.py | 89 +++++++++++++------ scripts/bump_mri_versions/python_lib.py | 29 ++++-- scripts/bump_mri_versions/readme.md | 44 ++++----- scripts/delete-jobs.sh | 6 +- scripts/rename-jobs.sh | 8 +- tox.ini | 1 - 15 files changed, 142 insertions(+), 95 deletions(-) diff --git a/jjb/docs/docs-post-rtd.sh b/jjb/docs/docs-post-rtd.sh index 6f5aa6be1..bc852bd4b 100644 --- a/jjb/docs/docs-post-rtd.sh +++ b/jjb/docs/docs-post-rtd.sh @@ -2,7 +2,7 @@ if [ "$GERRIT_BRANCH" = "master" ]; then RTD_BUILD_VERSION=latest else - RTD_BUILD_VERSION=$(echo $GERRIT_BRANCH | sed 's@/@-@g') + RTD_BUILD_VERSION=$(echo "$GERRIT_BRANCH" | sed 's@/@-@g') fi # shellcheck disable=SC1083 diff --git a/jjb/integration/common-functions.sh b/jjb/integration/common-functions.sh index e0cab0203..2a6476952 100644 --- a/jjb/integration/common-functions.sh +++ b/jjb/integration/common-functions.sh @@ -847,7 +847,7 @@ function get_nodes_list() { # Create the string for nodes for i in $(seq 1 "${NUM_ODL_SYSTEM}") ; do CONTROLLERIP=ODL_SYSTEM_${i}_IP - nodes[$i]=${!CONTROLLERIP} + nodes[i]=${!CONTROLLERIP} done nodes_list=$(join "${nodes[@]}") diff --git a/jjb/integration/integration-get-slave-addresses.sh b/jjb/integration/integration-get-slave-addresses.sh index 98cd3db7b..80eabb6fd 100644 --- a/jjb/integration/integration-get-slave-addresses.sh +++ b/jjb/integration/integration-get-slave-addresses.sh @@ -77,7 +77,7 @@ if [ ${#OPENSTACK_CONTROLLERS[@]} -eq 2 ]; then ctrl_index=$((ctrl_index -1)) tmp_addr=${OPENSTACK_SYSTEM[0]} OPENSTACK_SYSTEM[0]=${OPENSTACK_SYSTEM[$ctrl_index]} - OPENSTACK_SYSTEM[$ctrl_index]=$tmp_addr + OPENSTACK_SYSTEM[ctrl_index]=$tmp_addr fi # Add alias for ODL_SYSTEM_1_IP as ODL_SYSTEM_IP diff --git a/jjb/integration/integration-macros.yaml b/jjb/integration/integration-macros.yaml index c989a48fc..3043e4b59 100644 --- a/jjb/integration/integration-macros.yaml +++ b/jjb/integration/integration-macros.yaml @@ -512,4 +512,3 @@ done fi mark-unstable-if-failed: true - diff --git a/jjb/lf-infra-releasedockerhub.yaml b/jjb/lf-infra-releasedockerhub.yaml index 682f0f679..88f1f4279 100644 --- a/jjb/lf-infra-releasedockerhub.yaml +++ b/jjb/lf-infra-releasedockerhub.yaml @@ -3,10 +3,9 @@ name: lf-odl-releasedockerhub-macro builders: - shell: !include-raw-escape: - - ./run_releasedockerhub.sh + - ./run_releasedockerhub.sh ##- ../shell/run_releasedockerhub.sh "{org} {summary} {verbose} {copy}" - - job: name: lf-odl-release-docker-hub build-timeout: 90 @@ -14,55 +13,53 @@ project-type: freestyle node: ubuntu1804-docker-4c-4g disable-job: false - disabled: '{disable-job}' + disabled: "{disable-job}" # Runs once a day triggers: - - timed: '@daily' + - timed: "@daily" -# Define needed parameters for lf-infra-docker-login + # Define needed parameters for lf-infra-docker-login parameters: - string: name: DOCKER_REGISTRY - default: 'nexus3.opendaylight.org' + default: "nexus3.opendaylight.org" - string: name: REGISTRY_PORTS - default: '10002' + default: "10002" - string: name: DOCKERHUB_REGISTRY - default: 'docker.io' + default: "docker.io" -# Define needed parameters for lf-odl-releasedockerhub-macro + # Define needed parameters for lf-odl-releasedockerhub-macro - string: name: RELEASEDOCKERHUB_ORG - default: 'opendaylight' - description: '' + default: "opendaylight" + description: "" - string: name: RELEASEDOCKERHUB_REPO - default: '' - description: 'Only match repos that contains this string' + default: "" + description: "Only match repos that contains this string" - string: name: RELEASEDOCKERHUB_EXACT - default: '' - description: 'Repo has to exactly match RELEASEDOCKERHUB_REPO ' + default: "" + description: "Repo has to exactly match RELEASEDOCKERHUB_REPO " - string: name: RELEASEDOCKERHUB_SUMMARY - default: 'Yes' - description: 'Provides a brief statistic summary of what it found, and what it should do. Activate with Yes' + default: "Yes" + description: "Provides a brief statistic summary of what it found, and what it should do. Activate with Yes" - string: name: RELEASEDOCKERHUB_VERBOSE - default: '' - description: 'Provides a verbose output of the operations. Activate with Yes' + default: "" + description: "Provides a verbose output of the operations. Activate with Yes" - string: name: RELEASEDOCKERHUB_COPY - default: 'Yes' - description: 'If a copy operation should be done from Nexus3 to docker hub. Activate with Yes' - + default: "Yes" + description: "If a copy operation should be done from Nexus3 to docker hub. Activate with Yes" builders: - - lf-infra-pre-build ## shell/python-tools-install.sh + - lf-infra-pre-build ## shell/python-tools-install.sh - lf-infra-docker-login: - global-settings-file: 'global-settings' - settings-file: 'odl-releasedockerhub-authz-settings' + global-settings-file: "global-settings" + settings-file: "odl-releasedockerhub-authz-settings" - lf-odl-releasedockerhub-macro: - diff --git a/jjb/opendaylight-infra-deploy-maven-site.sh b/jjb/opendaylight-infra-deploy-maven-site.sh index bf674187c..bddcbe668 100644 --- a/jjb/opendaylight-infra-deploy-maven-site.sh +++ b/jjb/opendaylight-infra-deploy-maven-site.sh @@ -10,6 +10,7 @@ ############################################################################## # Ensure we fail the job if any steps fail. +# shellcheck disable=SC2039,SC3040 set -eu -o pipefail # shellcheck disable=SC1090 diff --git a/jjb/releng-maven-mri-stage.sh b/jjb/releng-maven-mri-stage.sh index 3b5ddd56d..523650211 100644 --- a/jjb/releng-maven-mri-stage.sh +++ b/jjb/releng-maven-mri-stage.sh @@ -8,6 +8,7 @@ echo "---> releng-maven-mri-stage.sh" # $STAGING_PROFILE_ID : Provided by a job parameter. # Ensure we fail the job if any steps fail. +# shellcheck disable=SC2039,SC3040 set -xeu -o pipefail TMP_FILE="$(mktemp)" diff --git a/jjb/releng-maven-mri-stage.yaml b/jjb/releng-maven-mri-stage.yaml index 8ddcf1bfd..092f02241 100644 --- a/jjb/releng-maven-mri-stage.yaml +++ b/jjb/releng-maven-mri-stage.yaml @@ -1,3 +1,4 @@ +--- #################### # COMMON FUNCTIONS # #################### diff --git a/jjb/run_releasedockerhub.sh b/jjb/run_releasedockerhub.sh index 474911ddd..d31391fe2 100644 --- a/jjb/run_releasedockerhub.sh +++ b/jjb/run_releasedockerhub.sh @@ -53,4 +53,5 @@ fi echo "cmd_str = >>$cmd_str<<" # Run the releasedockerhub command in lftools +# shellcheck disable=SC2086 lftools nexus docker releasedockerhub $cmd_str diff --git a/scripts/bump_mri_versions/main.py b/scripts/bump_mri_versions/main.py index bc69c66cc..dd1525212 100644 --- a/scripts/bump_mri_versions/main.py +++ b/scripts/bump_mri_versions/main.py @@ -3,28 +3,35 @@ # This program and the accompanying materials are made available under the # terms of the Eclipse Public License v1.0 which accompanies this distribution, # and is available at http://www.eclipse.org/legal/epl-v10.html +"""Main function for branch cutting a new stable release.""" import re import requests import python_lib + # pylint: disable=wrong-import-order from pathlib import Path from bs4 import BeautifulSoup -from lxml import etree + +# from lxml import etree +from defusedxml import lxml as etree + +"""Get the version from the groupId and artifactId.""" def get_version_for_artifact(group_id, artifact_id): + """Retrive version number from the groupId and artifactId.""" versions_list = [] - url = f'https://repo1.maven.org/maven2/org/opendaylight/{group_id}/{artifact_id}/' + url = f"https://repo1.maven.org/maven2/org/opendaylight/{group_id}/{artifact_id}/" response = requests.get(url).content - soup = BeautifulSoup(response, 'html.parser') + soup = BeautifulSoup(response, "html.parser") try: - html_lines = str(soup.find_all('pre')[0]).splitlines() + html_lines = str(soup.find_all("pre")[0]).splitlines() except IndexError: return "NOT FOUND" for line in html_lines: # Use a regular expression to find version - pattern = re.compile(r'\d+\.\d+\.\d+') + pattern = re.compile(r"\d+\.\d+\.\d+") title = pattern.search(line) try: versions_list.append(title.group()) @@ -34,61 +41,87 @@ def get_version_for_artifact(group_id, artifact_id): # get all xml files -for path in Path(python_lib.bumping_dir).rglob('*.xml'): +for path in Path(python_lib.bumping_dir).rglob("*.xml"): if "test/resources" in str(path): continue - tree = etree.parse(path) + parser = etree.XMLParser(resolve_entities=False, no_network=True) + tree = etree.parse(path, parser) root = tree.getroot() # update major and minor artifacts versions if "pom.xml" in str(path): prefix = "{" + root.nsmap[None] + "}" - # line above can trigger a 'KeyError: None' outside pom.xml and feature.xml files. - all_elements = tree.findall( - f'.//{prefix}parent') + tree.findall(f'.//{prefix}dependency') + # line above can trigger a 'KeyError: None' outside pom.xml and + # feature.xml files. + all_elements = tree.findall(f".//{prefix}parent") + tree.findall( + f".//{prefix}dependency" + ) for element in all_elements: - group_id_elem = (element.find(f'{prefix}groupId')) - artifact_id_elem = (element.find(f'{prefix}artifactId')) - version = (element.find(f'{prefix}version')) + group_id_elem = element.find(f"{prefix}groupId") + artifact_id_elem = element.find(f"{prefix}artifactId") + version = element.find(f"{prefix}version") try: if "org.opendaylight" in group_id_elem.text and version is not None: # skip artifacts containing items in skipped list skipped = ["${project.version}", "SNAPSHOT", "@project.version@"] if not any(x in version.text for x in skipped): new_version = get_version_for_artifact( - group_id_elem.text.split(".")[2], artifact_id_elem.text) + group_id_elem.text.split(".")[2], artifact_id_elem.text + ) if python_lib.check_minor_version(version, new_version): - print(python_lib.log_artifact( - path, group_id_elem, artifact_id_elem, version.text, new_version)) + print( + python_lib.log_artifact( + path, + group_id_elem, + artifact_id_elem, + version.text, + new_version, + ) + ) version.text = new_version - tree.write(path, encoding="UTF-8", pretty_print=True, - doctype='') + tree.write( + path, + encoding="UTF-8", + pretty_print=True, + doctype='', + ) except AttributeError: pass # update feature versions if "feature.xml" in str(path): prefix = "{" + root.nsmap[None] + "}" - # line above can trigger a 'KeyError: None' outside pom.xml and feature.xml files. - all_features = tree.findall(f'.//{prefix}feature') + # line above can trigger a 'KeyError: None' outside pom.xml and + # feature.xml files. + all_features = tree.findall(f".//{prefix}feature") # feature versions add +1 for feature in all_features: try: - if feature.attrib["version"] and feature.attrib["version"] != "${project.version}": + if ( + feature.attrib["version"] + and feature.attrib["version"] != "${project.version}" + ): current_version = feature.attrib["version"] # workaround for float feature versions - nums = current_version[1:-1].split(',') + nums = current_version[1:-1].split(",") if "." in nums[0]: nums[0] = str(round((float(nums[0]) + 0.01), 2)) else: nums[0] = str(int(nums[0]) + 1) nums[1] = str(int(nums[1]) + 1) - result = '[' + ','.join(nums) + ')' + result = "[" + ",".join(nums) + ")" feature.attrib["version"] = result - print(python_lib.log_artifact( - path=path, version=current_version, new_version=result)) - standalone = '' + print( + python_lib.log_artifact( + path=path, version=current_version, new_version=result + ) + ) + standalone = "" if tree.docinfo.standalone: standalone = ' standalone="yes"' - tree.write(path, encoding="UTF-8", pretty_print=True, - doctype=f'') + tree.write( + path, + encoding="UTF-8", + pretty_print=True, + doctype=f'', + ) except KeyError: pass diff --git a/scripts/bump_mri_versions/python_lib.py b/scripts/bump_mri_versions/python_lib.py index c72415479..31603edf8 100644 --- a/scripts/bump_mri_versions/python_lib.py +++ b/scripts/bump_mri_versions/python_lib.py @@ -4,36 +4,51 @@ # terms of the Eclipse Public License v1.0 which accompanies this distribution, # and is available at http://www.eclipse.org/legal/epl-v10.html +"""Library calls for branch cutting a new stable release.""" + # modify this dir for pick up project from there bumping_dir = "repos" def find_highest_revision(revisions): + """Return bigger versions in the tuple.""" # convert list of strings to list of tuples - converted_items = [tuple(map(int, item.split('.'))) for item in revisions] + converted_items = [tuple(map(int, item.split("."))) for item in revisions] biggest_item = max(converted_items, key=lambda x: x) - biggest_version = '.'.join(str(x) for x in biggest_item) + biggest_version = ".".join(str(x) for x in biggest_item) return biggest_version def log_artifact(path, group_id=None, artifact_id=None, version=None, new_version=None): + """Add filename and path, artifactId, versions to log.""" log = "" log += "XML FILE: " + str(path) + "\n" # if none, printing feature update if group_id is None: - log_line = ("path:", path, "VERSION:", version, - "NEW VERSION:", new_version) + log_line = ("path:", path, "VERSION:", version, "NEW VERSION:", new_version) # else printing artifact update else: - log_line = ("groupId:", group_id.text, "ARTIFACT ID:", - artifact_id.text, "VERSION:", version, "NEW VERSION:", new_version) + log_line = ( + "groupId:", + group_id.text, + "ARTIFACT ID:", + artifact_id.text, + "VERSION:", + version, + "NEW VERSION:", + new_version, + ) log += str(log_line) + "\n" log += str(100 * "*" + "\n") return log def check_minor_version(version, new_version): + """Take two version string and returns True if its same are the new version.""" # compares the corresponding elements of the two version strings - if any(int(elem_a) != int(elem_b) for elem_a, elem_b in zip(version.text.split("."), new_version.split("."))): + if any( + int(elem_a) != int(elem_b) + for elem_a, elem_b in zip(version.text.split("."), new_version.split(".")) + ): return True return False diff --git a/scripts/bump_mri_versions/readme.md b/scripts/bump_mri_versions/readme.md index 4659a3f42..4610d0202 100644 --- a/scripts/bump_mri_versions/readme.md +++ b/scripts/bump_mri_versions/readme.md @@ -1,50 +1,50 @@ + # Bumping MRI versions tool -This program is making versions changes in pom.xml files. For example 10.0.1 to 10.0.2 -The change will aply only if groupId.text contain "org.opendaylight". - -This program is also making changes in feature.xml files. For example [0.16,1) to [0.17,1) +This program is making versions changes in pom.xml files. For example 10.0.1 +to 10.0.2. The change will apply if groupId.text contain "org.opendaylight". +This program is also making changes in feature.xml files. For example +[0.16,1) to [0.17,1) ## Installing -*Prerequisite:* The followings are required for building test: +_Prerequisite:_ - Python 3.8+ GET THE CODE: USING HTTPS: - git clone "https://git.opendaylight.org/gerrit/releng/builder" +git clone "https://git.opendaylight.org/gerrit/releng/builder" USING SSH: - git clone "ssh://{USERNAME}@git.opendaylight.org:29418/releng/builder" +git clone "ssh://{USERNAME}@git.opendaylight.org:29418/releng/builder" NAVIGATE TO: - cd ~/builder/scripts/bump_mri_versions +cd ~/builder/scripts/bump_mri_versions INSTALL VIRTUAL ENVIROMENT PACKAGE: - sudo apt install python3-virtualenv +sudo apt install python3-virtualenv CREATE NEW VIRTUAL ENVIROMENT: - virtualenv venv +virtualenv venv ACTIVATE VIRTUAL ENVIROMENT: - . venv/bin/activate +. venv/bin/activate INSTALL LIBRARIES: - pip install requests bs4 lxml +pip install requests bs4 lxml SET FOLDER FOR TESTING: - clone repo for version updating in ~/builder/scripts/bump_mri_versions/repos or - update "bumping_dir" variable in python_lib.py file - +clone repo for version updating in ~/builder/scripts/bump_mri_versions/repos or +update "bumping_dir" variable in python_lib.py file ## Running @@ -53,7 +53,7 @@ RUN: python main.py ## Logs PRINT: - Every change will be printed out to the console. +All changes will be output to the console. examples here: @@ -63,4 +63,4 @@ PRINT: XML FILE: repos/ovsdb/southbound/southbound-features/odl-ovsdb-southbound-impl/src/main/feature/feature.xml ('path:', PosixPath('repos/ovsdb/southbound/southbound-features/odl-ovsdb-southbound-impl/src/main/feature/feature.xml'), 'VERSION:', '[4,5)', 'NEW VERSION:', '[5,6)') - **************************************************************************************************** \ No newline at end of file + **************************************************************************************************** diff --git a/scripts/delete-jobs.sh b/scripts/delete-jobs.sh index 7baf0a36d..e9263ca09 100755 --- a/scripts/delete-jobs.sh +++ b/scripts/delete-jobs.sh @@ -10,11 +10,11 @@ search_string=$1 -echo -n "Enter system (sandbox|releng): " +printf "Enter system (sandbox|releng): " read -r system -echo -n "Enter username: " +printf "Enter username: " read -r username -echo -n "Enter api_token: " +printf "Enter api_token: " read -r password echo "$username:$password" diff --git a/scripts/rename-jobs.sh b/scripts/rename-jobs.sh index 9187edf5a..a1cb96062 100755 --- a/scripts/rename-jobs.sh +++ b/scripts/rename-jobs.sh @@ -12,11 +12,11 @@ search_string=$1 replace_string=$2 -echo -n "Enter system (sandbox|releng): " +printf "Enter system (sandbox|releng): " read -r system -echo -n "Enter username: " +printf "Enter username: " read -r username -echo -n "Enter api_token: " +printf "Enter api_token: " read -r password echo "$username:$password" @@ -28,7 +28,7 @@ jobs=$(xmlstarlet sel -t -m '//hudson/job' \ grep "$search_string") for job in $(echo "$jobs" | tr "\n" " "); do - new_job=$(echo $job | sed "s@$search_string@$replace_string@g") + new_job=$(echo "$job" | sed "s@$search_string@$replace_string@g") echo "Renaming $job to $new_job" #curl --data "newName=${new_job}" "https://$username:$password@jenkins.opendaylight.org/$system/job/${job}/doRename" done diff --git a/tox.ini b/tox.ini index 7cd9bec94..56b21b345 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] minversion = 1.6 envlist = docs,docs-linkcheck,jjb-version,prefix,robot -# pre-commit skipsdist = true [testenv:docs] -- 2.36.6