From 930e108d454d34691474bd76f9d187913686a6f6 Mon Sep 17 00:00:00 2001 From: dongchengwei Date: Thu, 18 May 2023 17:47:25 +0800 Subject: [PATCH 1/3] [Feature] Test ci --- .github/workflows/bangc_ci.yaml | 29 +----- .../mlu_op_gtest/pb_gtest/src/zoo/nms/nms.cpp | 1 - ci.sh | 89 +++++++++++++++++++ combine_log.py | 44 +++++++++ file_guard.py | 30 +++++++ 5 files changed, 166 insertions(+), 27 deletions(-) create mode 100644 ci.sh create mode 100644 combine_log.py create mode 100644 file_guard.py diff --git a/.github/workflows/bangc_ci.yaml b/.github/workflows/bangc_ci.yaml index aa7325b89..a24e2a1ee 100644 --- a/.github/workflows/bangc_ci.yaml +++ b/.github/workflows/bangc_ci.yaml @@ -47,35 +47,12 @@ jobs: runner: [mlu370-m8] mlu_ops_version : [v0.7.0] cntoolkit_version : [cntoolkit3.5.0] - runs-on: ${{matrix.runner}} + runs-on: [self-hosted] steps: - uses: actions/checkout@v3 with: submodules: 'true' - - name: bangc_lint_check + - name: run_bangc_ops_ci run: > - docker run --rm -v $(pwd):/work -w /work docker-user.extrotec.com:30080/mlu-ops/mluops_ci:v0.2-x86_64-ubuntu16.04-BANGPy - ./tools/pre-commit origin/master - - - name: build_bangc_ops - run: > - docker run --rm -v $(pwd):/work -w /work docker-user.extrotec.com:30080/mlu-ops/mluops_ci:${{matrix.mlu_ops_version}}-devel-x86_64-ubuntu18.04-${{matrix.cntoolkit_version}} - ./build.sh --sub_module=bangc - - - name: bangc_ops_release_temp_cases - run: > - docker run --rm --device /dev/cambricon_ctl --device /dev/cambricon_dev0 --device /dev/commu0 - -v /testdata:/testdata -v $(pwd):/work -w /work docker-user.extrotec.com:30080/mlu-ops/mluops_ci:${{matrix.mlu_ops_version}}-devel-x86_64-ubuntu18.04-${{matrix.cntoolkit_version}} - ./test.sh --sub_module=bangc --cases_dir=/testdata/release_temp/default_platform - - - name: test_bangc_ops_release_temp_370_cases - if: matrix.runner == 'mlu370-m8' - run: > - docker run --rm --device /dev/cambricon_ctl --device /dev/cambricon_dev0 --device /dev/commu0 - -v /testdata:/testdata -v $(pwd):/work -w /work docker-user.extrotec.com:30080/mlu-ops/mluops_ci:${{matrix.mlu_ops_version}}-devel-x86_64-ubuntu18.04-${{matrix.cntoolkit_version}} - ./test.sh --sub_module=bangc --cases_dir=/testdata/release_temp/370 - - - name: clean - run: | - rm -rf bangc-ops/build + bash ci.sh diff --git a/bangc-ops/test/mlu_op_gtest/pb_gtest/src/zoo/nms/nms.cpp b/bangc-ops/test/mlu_op_gtest/pb_gtest/src/zoo/nms/nms.cpp index f5071ac98..e5b75bd51 100644 --- a/bangc-ops/test/mlu_op_gtest/pb_gtest/src/zoo/nms/nms.cpp +++ b/bangc-ops/test/mlu_op_gtest/pb_gtest/src/zoo/nms/nms.cpp @@ -427,7 +427,6 @@ void NmsExecutor::nms_detection_cpu( void NmsExecutor::cpuCompute() { assert(parser_->getInputNum() == 2); // assert(parser_->getOutputNum() == 1); - int max_output_boxes = parser_->getProtoNode()->nms_param().max_output_boxes(); float iou_thresh = parser_->getProtoNode()->nms_param().iou_threshold(); diff --git a/ci.sh b/ci.sh new file mode 100644 index 000000000..8096d11ec --- /dev/null +++ b/ci.sh @@ -0,0 +1,89 @@ +# /bin/bash +# get PR id +PR_string=$(echo $GITHUB_REF | grep -Eo "/[0-9]*/") +pr_id=(${PR_string//// }) + +# generate time stamp +current=`date "+%Y-%m-%d %H:%M:%S"` +timeStamp=`date -d "$current" +%s` +currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000)) + +# temporally set to mlu370 +card_type="MLU370-S4" + +# default repo name +repo_name="mlu-ops-dev" + +# repo ci root path +repo_root="/github_ci/${repo_name}_ci/" +if [ ! -d $repo_root ];then + mkdir $repo_root +fi + +# repo ci requests path +requests_path="$repo_root/requests" +if [ ! -d $requests_path ];then + mkdir $requests_path +fi + +# gen name of this ci +request_name="${repo_name}_${pr_id}_${currentTimeStamp}_${card_type}" + +# gen file and dir for this request +request_root="$repo_root/$request_name/" +sub_logs_path="$request_root/sub_logs/" + + +# echo "${repo_root}" +# echo "${requests_path}" +# echo "${request_root}" + +if [ ! -d $request_root ];then + mkdir $request_root +fi + +if [ ! -d $sub_logs_path ];then + mkdir $sub_logs_path +fi + +echo "working" > "$request_root/status" +chmod o+w "$request_root/status" + +if [ ! -f "$request_root/log" ];then + touch "$request_root/log" +fi + +chmod o+w "$request_root/log" + +if [ ! -f "$request_root/log_list" ];then + touch "$request_root/log_list" +fi + +chmod o+w "$request_root/log_list" + +# gen request file. +echo "${repo_name},${pr_id},${currentTimeStamp},${card_type}" > "$requests_path/${request_name}" + +# change dir group for server and client, or when server/client try to delete request, ftp may raise error. +chgrp -R ftpuser $request_root +chgrp -R ftpuser $requests_path + +# start script +python3 file_guard.py "$request_root/status" "$request_root/log" & +python3 combine_log.py "$request_root/log" "$request_root/log_list" "$request_root/sub_logs" "$request_root/status" & + +wait + +# status=$(cat ${request_root}/status) + +status=$( head -n +1 ${request_root}/status ) + +if [ "$status" != "success" ];then + return_info=$( sed -n 2p ${request_root}/status ) + echo "${return_info}" + exit -1 +else + return_info=$( sed -n 2p ${request_root}/status ) + echo "${return_info}" + exit 0 +fi diff --git a/combine_log.py b/combine_log.py new file mode 100644 index 000000000..78a94319b --- /dev/null +++ b/combine_log.py @@ -0,0 +1,44 @@ +import time +import sys +import os +# Get info. +# output_path: the target file that you want to combine sub log with. +# list_path: the list of sub log name. When it is updated, the correspondding file will be add to output tail. +# list_dir_path: the dir path where sub logs stored. +# status_path: the path of status file. When status file is written to "success" or "fail", exit script. + +output_path = sys.argv[1] +list_path = sys.argv[2] +list_dir_path = sys.argv[3] +status_path = sys.argv[4] + +if __name__ == '__main__': + # list_pos stores the last position that pointer of list file pointed to. + list_pos = 0 + while True: + list_file = open(list_path, 'r') + list_file.seek(list_pos) + # read all lines starting from list_pos. + items = list_file.readlines() + # update list_pos + list_pos = list_file.tell() + # if read any line + if items is not None: + items.sort() + for item in items: + sub_path = item.strip() + if sub_path is not "": + file_name = list_dir_path + '/' + sub_path + # while True: + if os.path.exists(file_name): + os.system('cat ' + file_name + ' >> ' + output_path) + # break + # check status_file, when read "success" or "fail" exit cycle, or else, sleep some seconds and start from beginning. + status_file = open(status_path) + status = status_file.readline().strip() + status_file.close() + if "fail" in status or "success" in status: + break + else: + time.sleep(2) + diff --git a/file_guard.py b/file_guard.py new file mode 100644 index 000000000..bbbc6a664 --- /dev/null +++ b/file_guard.py @@ -0,0 +1,30 @@ +import time +import sys +import os +guard_status_file = sys.argv[1] +guard_log_file = sys.argv[2] + +if __name__ == '__main__': + # where stores the last position that pointer pointed to. + where= 0 + while True: + file = open(guard_log_file, "r") + file.seek(where) + # if read any lines, call system echo to print each line. + for line in file.readlines(): + new_line = line.strip().replace("\'", "_").replace("\"", "_") + os.system('echo ' + "'%s'" % new_line) + # update where + where = file.tell() + file.close() + # check status, end process when read "success" or "fail" + status_file = open(guard_status_file, "r") + line = status_file.readline().strip() + status_file.close() + if "success" in line: + break + elif "fail" in line: + exit(-1) + # sleep for a while + time.sleep(2) + From 686149d370ebfed9980d3836716715c3d7da5599 Mon Sep 17 00:00:00 2001 From: dongchengwei Date: Wed, 5 Jul 2023 10:15:45 +0800 Subject: [PATCH 2/3] update ci.sh. --- ci.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci.sh b/ci.sh index 8096d11ec..42c09116c 100644 --- a/ci.sh +++ b/ci.sh @@ -1,5 +1,6 @@ # /bin/bash # get PR id +set -e PR_string=$(echo $GITHUB_REF | grep -Eo "/[0-9]*/") pr_id=(${PR_string//// }) @@ -15,7 +16,7 @@ card_type="MLU370-S4" repo_name="mlu-ops-dev" # repo ci root path -repo_root="/github_ci/${repo_name}_ci/" +repo_root="/home/cambricon/${repo_name}_ci/" if [ ! -d $repo_root ];then mkdir $repo_root fi @@ -65,8 +66,8 @@ chmod o+w "$request_root/log_list" echo "${repo_name},${pr_id},${currentTimeStamp},${card_type}" > "$requests_path/${request_name}" # change dir group for server and client, or when server/client try to delete request, ftp may raise error. -chgrp -R ftpuser $request_root -chgrp -R ftpuser $requests_path +chgrp -R cambricon $request_root +chgrp -R cambricon $requests_path # start script python3 file_guard.py "$request_root/status" "$request_root/log" & @@ -78,6 +79,8 @@ wait status=$( head -n +1 ${request_root}/status ) +set +e + if [ "$status" != "success" ];then return_info=$( sed -n 2p ${request_root}/status ) echo "${return_info}" From 788d43c4d8d9358fe562b6d02f29e2b0f4978c68 Mon Sep 17 00:00:00 2001 From: dongchengwei Date: Wed, 5 Jul 2023 10:19:27 +0800 Subject: [PATCH 3/3] a test --- ci.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci.sh b/ci.sh index 42c09116c..5f1b7124c 100644 --- a/ci.sh +++ b/ci.sh @@ -1,6 +1,7 @@ + + # /bin/bash # get PR id -set -e PR_string=$(echo $GITHUB_REF | grep -Eo "/[0-9]*/") pr_id=(${PR_string//// }) @@ -15,6 +16,8 @@ card_type="MLU370-S4" # default repo name repo_name="mlu-ops-dev" +pwd + # repo ci root path repo_root="/home/cambricon/${repo_name}_ci/" if [ ! -d $repo_root ];then @@ -79,8 +82,6 @@ wait status=$( head -n +1 ${request_root}/status ) -set +e - if [ "$status" != "success" ];then return_info=$( sed -n 2p ${request_root}/status ) echo "${return_info}"