diff --git a/bench.yaml b/bench.yaml new file mode 100644 index 0000000..64e699d --- /dev/null +++ b/bench.yaml @@ -0,0 +1,355 @@ +CMD_ARG: +- bench_args: + arg: perl -e 'print("hello\n")' + stdin_sh: sh + category: language + image: perl + repo: perl +- bench_args: + arg: perl6 -e 'print("hello\n")' + stdin_sh: sh + category: language + image: rakudo-star + repo: rakudo-star +- bench_args: + arg: pypy3 -c 'print("hello")' + stdin_sh: sh + category: language + image: pypy + repo: pypy +- bench_args: + arg: python -c 'print("hello")' + stdin_sh: sh + category: language + image: python + repo: python +- bench_args: + stdin_sh: sh + category: other + image: hello-world + repo: hello-world +CMD_ARG_WAIT: +- bench_args: + envs: + - key: MYSQL_ROOT_PASSWORD + value: abc + stdin_sh: sh + wait_line: 'mysqld: ready for connections' + category: database + image: mysql + repo: mysql +- bench_args: + envs: + - key: MYSQL_ROOT_PASSWORD + value: abc + stdin_sh: sh + wait_line: 'mysqld: ready for connections' + category: database + image: percona + repo: percona +- bench_args: + envs: + - key: MYSQL_ROOT_PASSWORD + value: abc + stdin_sh: sh + wait_line: 'mariadbd: ready for connections' + category: database + image: mariadb + repo: mariadb +- bench_args: + stdin_sh: sh + wait_line: database system is ready to accept connections + category: database + image: postgres + repo: postgres +- bench_args: + stdin_sh: sh + wait_line: Ready to accept connections + category: database + image: redis + repo: redis +- bench_args: + stdin_sh: sh + wait_line: started + category: database + image: crate + repo: crate +- bench_args: + stdin_sh: sh + wait_line: Server ready + category: database + image: rethinkdb + repo: rethinkdb +- bench_args: + stdin_sh: sh + wait_line: Listening for thrift clients + category: database + image: cassandra + repo: cassandra +- bench_args: + stdin_sh: sh + wait_line: waiting for connections + category: database + image: mongo + repo: mongo +- bench_args: + stdin_sh: sh + wait_line: '] started' + category: database + image: elasticsearch + repo: elasticsearch +- bench_args: + stdin_sh: sh + wait_line: Listening on + category: other + image: ghost + repo: ghost +- bench_args: + stdin_sh: sh + wait_line: apache2 -D FOREGROUND + category: other + image: drupal + repo: drupal +- bench_args: + stdin_sh: sh + wait_line: Jenkins is fully up and running + category: other + image: jenkins + repo: jenkins +- bench_args: + stdin_sh: sh + wait_line: Process[web] is up + category: other + image: sonarqube + repo: sonarqube +- bench_args: + stdin_sh: sh + wait_line: Server startup complete + category: other + image: rabbitmq + repo: rabbitmq +- bench_args: + stdin_sh: sh + wait_line: httpd -D FOREGROUND + category: web-server + image: httpd + repo: httpd +- bench_args: + stdin_sh: sh + wait_line: Running GlassFish + category: web-server + image: glassfish + repo: glassfish +- bench_args: + stdin_sh: sh + wait_line: 'main: Started' + category: web-server + image: jetty + repo: jetty +- bench_args: + stdin_sh: sh + wait_line: Zend Server started + category: web-server + image: php-zendserver + repo: php-zendserver +- bench_args: + arg: catalina.sh run + stdin_sh: sh + wait_line: Server startup + category: web-server + image: tomcat + repo: tomcat +CMD_STDIN: +- bench_args: + stdin: php -r "echo \"hello\n\";" + stdin_sh: sh + category: language + image: php + repo: php +- bench_args: + stdin: ruby -e "puts \"hello\"" + stdin_sh: sh + category: language + image: ruby + repo: ruby +- bench_args: + stdin: jruby -e "puts \"hello\"" + stdin_sh: sh + category: language + image: jruby + repo: jruby +- bench_args: + stdin: julia -e 'println("hello")' + stdin_sh: sh + category: language + image: julia + repo: julia +- bench_args: + mount: + - container_path: /go/src + host_path: misc/mount/go + stdin: cd /go/src; go run main.go + stdin_sh: sh + category: language + image: golang + repo: golang +- bench_args: + mount: + - container_path: /hello + host_path: misc/mount/clojure + stdin: cd /hello/hello; lein run + stdin_sh: sh + category: language + image: clojure + repo: clojure +- bench_args: + stdin: '"hello"' + stdin_sh: null + category: language + image: haskell + repo: haskell +- bench_args: + stdin: (print "hello") + stdin_sh: null + category: language + image: hylang + repo: hylang +- bench_args: + mount: + - container_path: /src + host_path: misc/mount/java + stdin: cd /src; javac Main.java; java Main + stdin_sh: sh + category: language + image: java + repo: java +- bench_args: + mount: + - container_path: /src + host_path: misc/mount/mono + stdin: cd /src; mcs main.cs; mono main.exe + stdin_sh: sh + category: language + image: mono + repo: mono +- bench_args: + stdin: sprintf("hello") + stdin_sh: R --no-save + category: language + image: r-base + repo: r-base +- bench_args: + mount: + - container_path: /src + host_path: misc/mount/gcc + stdin: cd /src; gcc main.c; ./a.out + stdin_sh: sh + category: language + image: gcc + repo: gcc +- bench_args: + mount: + - container_path: /src + host_path: misc/mount/thrift + stdin: cd /src; thrift --gen py hello.idl + stdin_sh: sh + category: language + image: thrift + repo: thrift +- bench_args: + stdin: sed -i "s/.cuda()//g" /benchmark/vision/test.py; sed -i "s/cuda/cpu/g" + /benchmark/vision/test.py; sed -i "/^ assert/d" /benchmark/vision/test.py; + sed -i "s/required=True/required=False/g" /benchmark/vision/test.py; sed -i + "s/20/1/g" /benchmark/vision/test.py; cd /benchmark; python /benchmark/vision/test.py + stdin_sh: sh + category: other + image: benchmark + repo: benchmark +- bench_args: + stdin: django-admin startproject hello + stdin_sh: sh + category: web-framework + image: django + repo: django +- bench_args: + stdin: rails new hello + stdin_sh: sh + category: web-framework + image: rails + repo: rails +CMD_URL_WAIT: +- bench_args: + envs: + - key: GUNICORN_OPTS + value: '["--preload"]' + stdin_sh: sh + wait_url: http://localhost:5000 + category: other + image: registry + repo: registry +- bench_args: + stdin_sh: sh + wait_url: http://localhost:80 + category: web-server + image: nginx + repo: nginx +- bench_args: + arg: node /src/index.js + mount: + - container_path: /src + host_path: misc/mount/node + stdin_sh: sh + wait_url: http://localhost:80 + category: web-framework + image: node + repo: node +- bench_args: + arg: iojs /src/index.js + mount: + - container_path: /src + host_path: misc/mount/iojs + stdin_sh: sh + wait_url: http://localhost:80 + category: web-framework + image: iojs + repo: iojs +ECHO_HELLO: +- category: distro + image: alpine + repo: alpine +- category: distro + image: busybox + repo: busybox +- category: distro + image: crux + repo: crux +- category: distro + image: cirros + repo: cirros +- category: distro + image: debian + repo: debian +- category: distro + image: ubuntu + repo: ubuntu +- category: distro + image: ubuntu-upstart + repo: ubuntu-upstart +- category: distro + image: ubuntu-debootstrap + repo: ubuntu-debootstrap +- category: distro + image: centos + repo: centos +- category: distro + image: fedora + repo: fedora +- category: distro + image: opensuse + repo: opensuse +- category: distro + image: oraclelinux + repo: oraclelinux +- category: distro + image: mageia + repo: mageia diff --git a/bench_ml.yaml b/bench_ml.yaml new file mode 100644 index 0000000..5a1b9f7 --- /dev/null +++ b/bench_ml.yaml @@ -0,0 +1,76 @@ +CMD_ARG: +- bench_args: + envs: + - key: NVIDIA_VISIBLE_DEVICES + value: 'all' + runtime: nvidia-container-runtime + shm_size: 8gb + category: machine-learning + image: chat-bench + repo: ml_platform + +CMD_ARG_WAIT: +- bench_args: + envs: + - key: NVIDIA_VISIBLE_DEVICES + value: 'all' + arg: ./test.sh + stdin_sh: sh + mount: + - container_path: /pytorch + host_path: misc/mount/pytorch + runtime: nvidia-container-runtime + shm_size: 8gb + work_dir: /pytorch + wait_line: 'benchmark end' + category: machine-learning + image: pytorch + repo: ml_platform +- bench_args: + envs: + - key: NVIDIA_VISIBLE_DEVICES + value: 'all' + arg: python tf_cnn_benchmarks.py --batch_size=32 --model=resnet152_v2 --variable_update=parameter_server + stdin_sh: sh + mount: + - container_path: /tf_cnn_benchmarks + host_path: misc/mount/tf_cnn_benchmarks + runtime: nvidia-container-runtime + shm_size: 8gb + work_dir: /tf_cnn_benchmarks + wait_line: 'total images' + category: machine-learning + image: tensorflow + repo: ml_platform +- bench_args: + envs: + - key: NVIDIA_VISIBLE_DEVICES + value: 'all' + arg: /opt/ml-platform/ddp.sh --num-iters 1000 + stdin_sh: sh + mount: + - container_path: /tf_cnn_benchmarks + host_path: misc/mount/tf_cnn_benchmarks + runtime: nvidia-container-runtime + shm_size: 8gb + wait_line: 'Img/sec' + category: machine-learning + image: tensorflow + repo: ml_platform + +CMD_URL_WAIT: +- bench_args: + envs: + - key: NVIDIA_VISIBLE_DEVICES + value: 'all' + arg: tritonserver --model-repository=/models + stdin_sh: sh + mount: + - container_path: /models + host_path: misc/mount/model_repository + runtime: nvidia-container-runtime + shm_size: 8gb + wait_url: http://127.0.0.1:8000/v2/health/ready + category: machine-learning + image: tritonserver + repo: ml_platform \ No newline at end of file diff --git a/draw.py b/draw.py index 4662012..622b1a9 100755 --- a/draw.py +++ b/draw.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python3 +#!python3 import os import json @@ -29,6 +29,7 @@ def to_csv(): average_list = [] + image_size = dict() for current_dir, _, file_list in os.walk(data_dir): for filename in file_list: filename_path = os.path.join(current_dir, filename) @@ -38,16 +39,17 @@ def to_csv(): if line.strip() != "": json_line = json.loads(line) image = json_line["bench"] - pull_time = json_line["pull_time"] - create_time = json_line["create_time"] - run_time = json_line["run_time"] + pull_elapsed = json_line["pull_elapsed"] + create_elapsed = json_line["create_elapsed"] + run_elapsed = json_line["run_elapsed"] + image_size[image] = json_line["image_size"] average_list.append( { "image": image, - "pull": pull_time, - "create": create_time, - "run": run_time, + "pull": float(pull_elapsed), + "create": float(create_elapsed), + "run": float(run_elapsed), } ) @@ -103,8 +105,8 @@ def to_csv(): if os.path.exists(result_dir): shutil.rmtree(result_dir, ignore_errors=True) os.mkdir(result_dir) - os.mkdir(os.path.join(result_dir, "/", sub_data_dir)) - os.mkdir(os.path.join(result_dir, "/", sub_picture_dir)) + os.mkdir(result_dir + "/" + sub_data_dir) + os.mkdir(result_dir + "/" + sub_picture_dir) type_order = CategoricalDtype(["pull", "create", "run"], ordered=True) all_data_pd_line = [] @@ -114,7 +116,7 @@ def to_csv(): data_pd["type"] = data_pd["type"].astype(type_order) data_pd.sort_values(by="type", inplace=True, ascending=True) print(key, data_pd) - data_pd.to_csv(os.path.join(result_dir, "/", sub_data_dir, "/", key, ".csv")) + data_pd.to_csv(result_dir + "/" + sub_data_dir + "/" + key + ".csv") for image_name, image_data in data_pd.groupby("image"): all_data_pd_line = all_data_pd_line + [ @@ -123,21 +125,18 @@ def to_csv(): "pull": image_data[image_data["type"] == "pull"]["mean"].mean(), "create": image_data[image_data["type"] == "create"]["mean"].mean(), "run": image_data[image_data["type"] == "run"]["mean"].mean(), + "size": image_size[image_name], } ] all_data_pd = pd.DataFrame(all_data_pd_line) - all_data_pd.to_csv(os.path.join(result_dir, "/", "all_mean.csv")) + all_data_pd.to_csv(result_dir + "/" + "all_mean.csv") def draw(): - if os.path.exists(os.path.join(result_dir, "/", sub_picture_dir)): - shutil.rmtree( - os.path.join(result_dir, "/", sub_picture_dir), ignore_errors=True - ) - os.mkdir(os.path.join(result_dir, "/", sub_picture_dir)) - for current_dir, _, file_list in os.walk( - os.path.join(result_dir, "/", sub_data_dir) - ): + if os.path.exists(result_dir + "/" + sub_picture_dir): + shutil.rmtree(result_dir + "/" + sub_picture_dir, ignore_errors=True) + os.mkdir(result_dir + "/" + sub_picture_dir) + for current_dir, _, file_list in os.walk(result_dir + "/" + sub_data_dir): for filename in file_list: filename_path = os.path.join(current_dir, filename) print("file: ", filename_path) @@ -145,12 +144,12 @@ def draw(): print(data_pd) for index, data_series in data_pd.iterrows(): - picture_path = os.path.join( - result_dir, - "/", - sub_picture_dir, - "/", - data_series["image"].split(":")[0], + picture_path = ( + result_dir + + "/" + + sub_picture_dir + + "/" + + data_series["image"].split(":")[0] ) if not os.path.exists(picture_path): os.mkdir(picture_path) @@ -178,21 +177,23 @@ def draw(): plt.xlabel(None) plt.subplots_adjust(left=0.1, bottom=0.1, right=0.9, top=0.9) plt.savefig( - os.path.join( - picture_path, - "/", - data_series["image"].replace(":", "-"), - "_", - data_series["type"], - ".png", - ) + picture_path + + "/" + + data_series["image"].replace(":", "-") + + "_" + + data_series["type"] + + ".png" ) +def image_info(row): + tag = "latest" if len(row["image"].split(":")) <= 1 else row["image"].split(":")[1] + size = row["size"] + return f"{tag}\n({size})" + + def draw_all(): - all_data_pd = pd.read_csv( - os.path.join(result_dir, "/", "all_mean.csv"), index_col=0 - ) + all_data_pd = pd.read_csv(result_dir + "/" + "all_mean.csv", index_col=0) print(all_data_pd) all_data = dict() @@ -203,6 +204,7 @@ def draw_all(): "pull": data["pull"].mean(), "create": data["create"].mean(), "run": data["run"].mean(), + "size": data["size"].item(), } ] @@ -215,6 +217,9 @@ def draw_all(): for key in all_data: data_pd = pd.DataFrame(all_data[key]) fig, ax = plt.subplots() + + data_pd["image"] = data_pd.apply(lambda row: image_info(row), axis=1) + print(data_pd) ax.bar(data_pd["image"], data_pd["pull"], label="pull") ax.bar( @@ -228,10 +233,11 @@ def draw_all(): ) ax.legend(bbox_to_anchor=(1.26, 1)) + ax.set_title(key) plt.subplots_adjust(left=0.12, bottom=0.32, right=0.798, top=0.88) - plt.xticks(rotation=45) + plt.xticks(rotation=20) plt.ylabel("time(s)") - plt.savefig(os.path.join(result_dir, "/", key, ".png")) + plt.savefig(result_dir + "/" + key + ".png") if __name__ == "__main__": diff --git a/generate_image.sh b/generate_image.sh new file mode 100755 index 0000000..301523e --- /dev/null +++ b/generate_image.sh @@ -0,0 +1,427 @@ +#!/bin/bash + +tmpdir=tmp +accessed_list_dir=accessed_list +registry=docker.io/library + +image_list=images.txt +target_image_list=generated_image_list.txt +target_bench_yaml=generated_bench.yaml +file_pattern=lbr + +function large_number_small_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + if [ ! -d ${accessed_list_dir} ]; then + mkdir ${accessed_list_dir} + fi + file_number=(128 256 512 1024 2048 4096 8192 16384) + file_size=(1 2 4 8 16 32 64 128) + layer_number=(1 2 4 8 16 32 64) + for number in ${file_number[@]}; do + for size in ${file_size[@]}; do + for layer in ${layer_number[@]}; do + dir=${tmpdir}/dir_number${number}_size${size}_layer${layer} + mkdir -p ${dir} + + kb=$((${number} * ${size})) + kbplayer=$((${kb} / ${layer})) + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, size: ${size}, layer: ${layer}, total ${kb}KB, per layer: ${kbplayer}KB" + + image_name=small-file-s${size}:n${number}-l${layer} + image=${registry}/${image_name} + for i in $(cat ${image_list}); do + if [[ "${i}" == "${image}" ]]; then + echo "Skip image ${image}." + continue 2 + fi + done + echo "generating image ${image}..." + image_accessed_list=${accessed_list_dir}/${image_name} + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1K count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + cp ${image_dir}/file_list_path.txt ${image_accessed_list} + cp ${image_dir}/file_list_path_shuffed.txt ${image_accessed_list}.shuffed + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function small_number_large_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + if [ ! -d ${accessed_list_dir} ]; then + mkdir ${accessed_list_dir} + fi + file_number=(1 2 4 8) + file_size=(1 2 4 8) + layer_number=(1 2) + for number in ${file_number[@]}; do + for size in ${file_size[@]}; do + for layer in $(seq ${number}); do + if [[ $((${number} % ${layer})) != 0 ]]; then + echo "skip number: ${number}, layer: ${layer}" + continue + fi + dir=${tmpdir}/dir_number${number}_size${size}_layer${layer} + mkdir -p ${dir} + + gb=$((${number} * ${size})) + kb=$((${number} * ${size} * ${gb} * 1024 * 1024)) + kbplayer=$((${kb} / ${layer})) + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, size: ${size}, layer: ${layer}, total ${kb}KB, per layer: ${kbplayer}KB" + + image_name=large-file-s${size}:n${number}-l${layer} + image=${registry}/${image_name} + for i in $(cat ${image_list}); do + if [[ "${i}" == "${image}" ]]; then + echo "Skip image ${image}." + continue 2 + fi + done + echo "generating image ${image}..." + image_accessed_list=${accessed_list_dir}/${image_name} + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1G count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + cp ${image_dir}/file_list_path.txt ${image_accessed_list} + cp ${image_dir}/file_list_path_shuffed.txt ${image_accessed_list}.shuffed + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function large_number_random_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + if [ ! -d ${accessed_list_dir} ]; then + mkdir ${accessed_list_dir} + fi + layer_number=(1 2 4 8 16 32 64) + for layer in ${layer_number[@]}; do + for i in $(seq 5); do + number=$(((RANDOM % 65536) + 100)) + for j in $(seq 2); do + dir=${tmpdir}/dir_number${i}_size${j}_layer${layer} + mkdir -p ${dir} + + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, layer: ${layer}, fileplayer: ${fileplayer}" + + image_name=random-file-${j}:n${number}-l${layer} + image=${registry}/${image_name} + for i in $(cat ${image_list}); do + if [[ "${i}" == "${image}" ]]; then + echo "Skip image ${image}." + continue 2 + fi + done + echo "generating image ${image}..." + image_accessed_list=${accessed_list_dir}/${image_name} + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + size=$(((RANDOM % 128) + 1)) + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1K count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + cp ${image_dir}/file_list_path.txt ${image_accessed_list} + cp ${image_dir}/file_list_path_shuffed.txt ${image_accessed_list}.shuffed + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function large_base_image_random_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + if [ ! -d ${accessed_list_dir} ]; then + mkdir ${accessed_list_dir} + fi + + file_number=(1 2 4) + file_size=(1 2 4) + for number in ${file_number[@]}; do + for size in ${file_size[@]}; do + dir=${tmpdir}/dir_number${number}_size${size} + mkdir -p ${dir} + + mb=$((${size} * 1024)) + total_gb=$((${number} * ${size})) + total_mb=$((${number} * ${mb})) + echo "number: ${number}, size: ${size}, total ${total_gb}GB" + + base_image=${registry}/nydus-test-base:n${number}-s${size} + echo "generate base images ${base_image}..." + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + base_dir=${dir}/base + mkdir -p ${base_dir} + for f in $(seq ${number}); do + file_name=${base_dir}/file_${f} + echo "dd if=/dev/urandom of=${file_name} bs=1M count=${total_mb} conv=notrunc" + dd if=/dev/urandom of=${file_name} bs=1M count=${total_mb} conv=notrunc >/dev/null 2>&1 + echo "/tmp/base/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY base \/tmp\/base\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + + sed -i "/^$/d" ${dir}/file_list_path.txt + + sed -i "/^ADD file_list_path_shuffed.txt.*/d" ${dir}/Dockerfile + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + sed -i "/^ENTRYPOINT.*/d" ${dir}/Dockerfile + sed -i "/^ADD entrypoint.sh.*/d" ${dir}/Dockerfile + + layer_number=(1 2 4) + for layer in ${layer_number[@]}; do + for i in $(seq 2); do + image_file_number=$(((RANDOM % 8192) + 64)) + + image_dir=${dir}/dir_base_n${number}_s${size}_upper_number${i}_size${j}_layer${layer} + mkdir -p ${image_dir} + + fileplayer=$((${image_file_number} / ${layer})) + echo "image_file_number: ${image_file_number}, layer: ${layer}, fileplayer: ${fileplayer}" + + image_name=base-n${number}-s${size}-f${image_file_number}-l${layer} + image=${registry}/${image_name} + for j in $(cat ${image_list}); do + if [[ "${i}" == "${image}" ]]; then + echo "Skip image ${image}." + continue 2 + fi + done + echo "generating image ${image}..." + image_accessed_list=${accessed_list_dir}/${image_name} + + cp template/Dockerfile ${image_dir} + cp ${dir}/file_list_path.txt ${image_dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${image_dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + image_size=$(((RANDOM % 1024) + 1)) + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1K count=${image_size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${image_dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${image_dir}/Dockerfile + done + + sed -i "/^$/d" ${image_dir}/file_list_path.txt + shuf ${image_dir}/file_list_path.txt -o ${image_dir}/file_list_path_shuffed.txt + cp ${image_dir}/file_list_path.txt ${image_accessed_list} + cp ${image_dir}/file_list_path_shuffed.txt ${image_accessed_list}.shuffed + + sed -i "/^COPY REPLACE_ME.*/d" ${image_dir}/Dockerfile + cp template/entrypoint.sh ${image_dir} + + sed "s|^FROM bash:latest|FROM ${base_image}|g" ${image_dir}/Dockerfile + + sudo docker image load -i bash.latest + sudo docker build -t ${base_image} ${dir} + + sudo docker build -t ${image} ${image_dir} + + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${image_dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + + done + done + + rm -rf ${dir} + done + done +} + +function generate_bench_yaml() { + echo "" >${target_image_list} + cp template/bench.yaml ${target_bench_yaml} + images=($(cat ${image_list} | tr "\n" " ")) + for image in ${images[@]}; do + repo=$(echo ${image} | awk -F\/ '{print $(NF-1)}') + image_name=$(echo ${image} | awk -F\/ '{print $(NF)}') + + echo ${image_name} >>${target_image_list} + + name=$(echo ${image_name} | awk -F: '{print $1}') + if [[ "${tag}" == "" ]]; then + tag=latest + fi + sed -i "s/^ REPLACE_ME/- bench_args:\n wait_line: 'Read file list done'\n arg: seq \/file_list_path_shuffed.txt\n category: test\n image: ${name}\n repo: ${repo}\n REPLACE_ME/g" ${target_bench_yaml} + done + sed -i "/REPLACE_ME/d" ${target_bench_yaml} +} + +######################################################### +# Usage information +# Globals: +# None +# Arguments: +# None +# Returns: +# None +######################################################### +function usage() { + echo "Usage:" + echo -e "$0 [-t TARGET_REGISTRY] [-p TARGET_IMAGE_LIST_PATH] +[-t target registry] \target registry for pushing image +[-p target image list path] \tfile path that contains images list (line by line) +[-b target bench yaml path] \tfile path for bench configuration +[-f generated file pattern] \tfile pattern to be generated, available [ + \t\"ls\"\t(large number of small size files) + \t\"sl\"\t(small number of large size files) + \t\"lr\"\t(large number of random size files) + \t\"lbr\"\t(random number of random size files with large size base image) + \t\"all\"\t(all of the above) ] +" + exit -1 +} + +available_operation="ls sl lr lbr all" + +while getopts p:t:b:f:h OPT; do + case $OPT in + t) + registry=${OPTARG} + ;; + p) + target_image_list=${OPTARG} + ;; + b) + target_bench_yaml=${OPTARG} + ;; + f) + pattern=${OPTARG} + if ! [[ "$available_pattern" =~ "$pattern" ]]; then + echo "file pattern ${pattern} not support, use default value [${file_pattern}]" + else + file_pattern=${pattern} + fi + ;; + *) + usage + ;; + esac +done +shift $((OPTIND - 1)) + +echo "target registry: ${registry}" +echo "target image list: ${target_image_list}" +echo "target bench yaml: ${target_bench_yaml}" + +if [[ ! -f ${image_list} ]]; then + touch ${image_list} +fi +sudo docker pull bash:latest +sudo docker image save bash:latest -o bash.latest + +if [[ ${file_pattern} == "ls" ]]; then + large_number_small_files +fi +if [[ ${file_pattern} == "sl" ]]; then + small_number_large_files +fi +if [[ ${file_pattern} == "lr" ]]; then + large_number_random_files +fi +if [[ ${file_pattern} == "lbr" ]]; then + large_base_image_random_files +fi +if [[ ${file_pattern} == "all" ]]; then + large_number_small_files + small_number_large_files + large_number_random_files + large_base_image_random_files +fi + +generate_bench_yaml diff --git a/hello.py b/hello.py index 87d1a2d..74dac28 100755 --- a/hello.py +++ b/hello.py @@ -30,11 +30,8 @@ from argparse import ArgumentParser from datetime import datetime from contextlib import contextmanager +import yaml -NGINX_PORT = 20000 -IOJS_PORT = 20001 -NODE_PORT = 20002 -REGISTRY_PORT = 20003 TMP_DIR = tempfile.mkdtemp() @@ -135,7 +132,17 @@ def timer(cmd): class RunArgs: def __init__( - self, env={}, arg="", stdin="", stdin_sh="sh", waitline="", mount=[], waitURL="" + self, + env={}, + arg="", + stdin="", + stdin_sh="sh", + waitline="", + mount=[], + waitURL="", + runtime="", + shmSize="", + workDir="", ): self.env = env self.arg = arg @@ -144,6 +151,9 @@ def __init__( self.waitline = waitline self.mount = mount self.waitURL = waitURL + self.runtime = runtime + self.shmSize = shmSize + self.workDir = workDir class Docker: @@ -235,178 +245,15 @@ def set_tag(self, tag): self.name = f"{self.name}:{tag}" -class BenchRunner: - ECHO_HELLO = set( - [ - "alpine", - "busybox", - "crux", - "cirros", - "debian", - "ubuntu", - "ubuntu-upstart", - "ubuntu-debootstrap", - "centos", - "fedora", - "opensuse", - "oraclelinux", - "mageia", - ] - ) +class BenchResult: + def __init__(self, pull_elapsed, create_elapsed, run_elapsed, size): + self.pull_elapsed = pull_elapsed + self.create_elapsed = create_elapsed + self.run_elapsed = run_elapsed + self.size = size - CMD_ARG_WAIT = { - "mysql": RunArgs( - env={"MYSQL_ROOT_PASSWORD": "abc"}, waitline="mysqld: ready for connections" - ), - "percona": RunArgs( - env={"MYSQL_ROOT_PASSWORD": "abc"}, waitline="mysqld: ready for connections" - ), - "mariadb": RunArgs( - env={"MYSQL_ROOT_PASSWORD": "abc"}, - waitline="mariadbd: ready for connections", - ), - "postgres": RunArgs(waitline="database system is ready to accept connections"), - "redis": RunArgs(waitline="Ready to accept connections"), - "crate": RunArgs(waitline="started"), - "rethinkdb": RunArgs(waitline="Server ready"), - "ghost": RunArgs(waitline="Listening on"), - "glassfish": RunArgs(waitline="Running GlassFish"), - "drupal": RunArgs(waitline="apache2 -D FOREGROUND"), - "elasticsearch": RunArgs(waitline="] started"), - "cassandra": RunArgs(waitline="Listening for thrift clients"), - "httpd": RunArgs(waitline="httpd -D FOREGROUND"), - "jenkins": RunArgs(waitline="Jenkins is fully up and running"), - "jetty": RunArgs(waitline="main: Started"), - "mongo": RunArgs(waitline="waiting for connections"), - "php-zendserver": RunArgs(waitline="Zend Server started"), - "rabbitmq": RunArgs(waitline="Server startup complete"), - "sonarqube": RunArgs(waitline="Process[web] is up"), - "tomcat": RunArgs(arg="catalina.sh run", waitline="Server startup"), - } - - CMD_STDIN = { - "php": RunArgs(stdin='php -r "echo \\"hello\\n\\";"'), - "ruby": RunArgs(stdin='ruby -e "puts \\"hello\\""'), - "jruby": RunArgs(stdin='jruby -e "puts \\"hello\\""'), - "julia": RunArgs(stdin="julia -e 'println(\"hello\")'"), - "gcc": RunArgs(stdin="cd /src; gcc main.c; ./a.out", mount=[("gcc", "/src")]), - "golang": RunArgs( - stdin="cd /go/src; go run main.go", mount=[("go", "/go/src")] - ), - "clojure": RunArgs( - stdin="cd /hello/hello; lein run", mount=[("clojure", "/hello")] - ), - "django": RunArgs(stdin="django-admin startproject hello"), - "rails": RunArgs(stdin="rails new hello"), - "haskell": RunArgs(stdin='"hello"', stdin_sh=None), - "hylang": RunArgs(stdin='(print "hello")', stdin_sh=None), - "java": RunArgs( - stdin="cd /src; javac Main.java; java Main", mount=[("java", "/src")] - ), - "mono": RunArgs( - stdin="cd /src; mcs main.cs; mono main.exe", mount=[("mono", "/src")] - ), - "r-base": RunArgs(stdin='sprintf("hello")', stdin_sh="R --no-save"), - "thrift": RunArgs( - stdin="cd /src; thrift --gen py hello.idl", mount=[("thrift", "/src")] - ), - "benchmark": RunArgs( - stdin='sed -i "s/.cuda()//g" /benchmark/vision/test.py; sed -i "s/cuda/cpu/g" /benchmark/vision/test.py; sed -i "/^ assert/d" /benchmark/vision/test.py; sed -i "s/required=True/required=False/g" /benchmark/vision/test.py; sed -i "s/20/1/g" /benchmark/vision/test.py; cd /benchmark; python /benchmark/vision/test.py' - ), - } - - CMD_ARG = { - "perl": RunArgs(arg="perl -e 'print(\"hello\\n\")'"), - "rakudo-star": RunArgs(arg="perl6 -e 'print(\"hello\\n\")'"), - "pypy": RunArgs(arg="pypy3 -c 'print(\"hello\")'"), - "python": RunArgs(arg="python -c 'print(\"hello\")'"), - "hello-world": RunArgs(), - } - - CMD_URL_WAIT = { - "nginx": RunArgs(waitURL="http://localhost:80"), - "iojs": RunArgs( - arg="iojs /src/index.js", - mount=[("iojs", "/src")], - waitURL="http://localhost:80", - ), - "node": RunArgs( - arg="node /src/index.js", - mount=[("node", "/src")], - waitURL="http://localhost:80", - ), - "registry": RunArgs( - env={"GUNICORN_OPTS": '["--preload"]'}, waitURL="http://localhost:5000" - ), - } - - # complete listing - ALL = dict( - [ - (b.name, b) - for b in [ - Bench("alpine", "distro"), - Bench("busybox", "distro"), - Bench("crux", "distro"), - Bench("cirros", "distro"), - Bench("debian", "distro"), - Bench("ubuntu", "distro"), - Bench("ubuntu-upstart", "distro"), - Bench("ubuntu-debootstrap", "distro"), - Bench("centos", "distro"), - Bench("fedora", "distro"), - Bench("opensuse", "distro"), - Bench("oraclelinux", "distro"), - Bench("mageia", "distro"), - Bench("mysql", "database"), - Bench("percona", "database"), - Bench("mariadb", "database"), - Bench("postgres", "database"), - Bench("redis", "database"), - Bench("crate", "database"), - Bench("rethinkdb", "database"), - Bench("php", "language"), - Bench("ruby", "language"), - Bench("jruby", "language"), - Bench("julia", "language"), - Bench("perl", "language"), - Bench("rakudo-star", "language"), - Bench("pypy", "language"), - Bench("python", "language"), - Bench("golang", "language"), - Bench("clojure", "language"), - Bench("haskell", "language"), - Bench("hylang", "language"), - Bench("java", "language"), - Bench("mono", "language"), - Bench("r-base", "language"), - Bench("gcc", "language"), - Bench("thrift", "language"), - Bench("benchmark"), - Bench("cassandra", "database"), - Bench("mongo", "database"), - Bench("elasticsearch", "database"), - Bench("hello-world"), - Bench("ghost"), - Bench("drupal"), - Bench("jenkins"), - Bench("sonarqube"), - Bench("rabbitmq"), - Bench("registry"), - Bench("httpd", "web-server"), - Bench("nginx", "web-server"), - Bench("glassfish", "web-server"), - Bench("jetty", "web-server"), - Bench("php-zendserver", "web-server"), - Bench("tomcat", "web-server"), - Bench("django", "web-framework"), - Bench("rails", "web-framework"), - Bench("node", "web-framework"), - Bench("iojs", "web-framework"), - ] - ] - ) +class BenchRunner: def __init__( self, docker="docker", @@ -415,6 +262,7 @@ def __init__( snapshotter="overlayfs", cleanup=True, insecure_registry=False, + bench_config="bench.yaml", ): self.registry = registry if self.registry != "": @@ -430,6 +278,126 @@ def __init__( if "nerdctl" == docker: self.docker.set_snapshotter(snapshotter) self.cleanup = cleanup + self.bench_config = bench_config + + def load_bench_config(self): + bench_config = self.bench_config + print(f"Loading bench configuration from {bench_config}...") + with open(bench_config, "r") as stream: + data = yaml.safe_load(stream) + + echo_hello_runner = set() + echo_hello = dict() + if "ECHO_HELLO" in data: + for line in data["ECHO_HELLO"]: + name = line["image"] + echo_hello_runner.add(name) + echo_hello[name] = Bench(name, line["category"]) + + cmd_arg_wait_runner = dict() + cmd_arg_wait = dict() + if "CMD_ARG_WAIT" in data: + for line in data["CMD_ARG_WAIT"]: + name = line["image"] + args = line["bench_args"] + print(f"CMD_ARG_WAIT image: {name}, args: {args}") + cmd_arg_wait_runner[name] = RunArgs( + env=dict([(item["key"], item["value"]) for item in args["envs"]]) + if "envs" in args + else {}, + waitline=args["wait_line"] if "wait_line" in args else "", + mount=[(m["host_path"], m["container_path"]) for m in args["mount"]] + if "mount" in args + else [], + arg=args["arg"] if "arg" in args else "", + stdin=args["stdin"] if "stdin" in args else "", + stdin_sh=args["stdin_sh"] if "stdin_sh" in args else "", + runtime=args["runtime"] if "runtime" in args else "", + shmSize=args["shm_size"] if "shm_size" in args else "", + workDir=args["work_dir"] if "work_dir" in args else "", + ) + cmd_arg_wait[name] = Bench(name, line["category"]) + + cmd_stdin_runner = dict() + cmd_stdin = dict() + if "CMD_STDIN" in data: + for line in data["CMD_STDIN"]: + name = line["image"] + args = line["bench_args"] + print(f"CMD_STDIN image: {name}, args: {args}") + cmd_stdin_runner[name] = RunArgs( + env=dict([(item["key"], item["value"]) for item in args["envs"]]) + if "envs" in args + else {}, + mount=[(m["host_path"], m["container_path"]) for m in args["mount"]] + if "mount" in args + else [], + arg=args["arg"] if "arg" in args else "", + stdin=args["stdin"] if "stdin" in args else "", + stdin_sh=args["stdin_sh"] if "stdin_sh" in args else "", + runtime=args["runtime"] if "runtime" in args else "", + shmSize=args["shm_size"] if "shm_size" in args else "", + workDir=args["work_dir"] if "work_dir" in args else "", + ) + cmd_stdin[name] = Bench(name, line["category"]) + + cmd_arg_runner = dict() + cmd_arg = dict() + if "CMD_ARG" in data: + for line in data["CMD_ARG"]: + name = line["image"] + args = line["bench_args"] + print(f"CMD_ARG image: {name}, args: {args}") + cmd_arg_runner[name] = RunArgs( + env=dict([(item["key"], item["value"]) for item in args["envs"]]) + if "envs" in args + else {}, + mount=[(m["host_path"], m["container_path"]) for m in args["mount"]] + if "mount" in args + else [], + arg=args["arg"] if "arg" in args else "", + stdin=args["stdin"] if "stdin" in args else "", + stdin_sh=args["stdin_sh"] if "stdin_sh" in args else "", + runtime=args["runtime"] if "runtime" in args else "", + shmSize=args["shm_size"] if "shm_size" in args else "", + workDir=args["work_dir"] if "work_dir" in args else "", + ) + cmd_arg[name] = Bench(name, line["category"]) + + cmd_url_wait_runner = dict() + cmd_url_wait = dict() + if "CMD_URL_WAIT" in data: + for line in data["CMD_URL_WAIT"]: + name = line["image"] + args = line["bench_args"] + print(f"CMD_URL_WAIT image: {name}, args: {args}") + cmd_url_wait_runner[name] = RunArgs( + env=dict([(item["key"], item["value"]) for item in args["envs"]]) + if "envs" in args + else {}, + waitURL=args["wait_url"] if "wait_url" in args else "", + mount=[(m["host_path"], m["container_path"]) for m in args["mount"]] + if "mount" in args + else [], + arg=args["arg"] if "arg" in args else "", + stdin=args["stdin"] if "stdin" in args else "", + stdin_sh=args["stdin_sh"] if "stdin_sh" in args else "", + runtime=args["runtime"] if "runtime" in args else "", + shmSize=args["shm_size"] if "shm_size" in args else "", + workDir=args["work_dir"] if "work_dir" in args else "", + ) + cmd_url_wait[name] = Bench(name, line["category"]) + + all = {**echo_hello, **cmd_arg_wait, **cmd_stdin, **cmd_arg, **cmd_url_wait} + print([name for name in all.keys()]) + + self.ECHO_HELLO = echo_hello_runner + self.CMD_ARG_WAIT = cmd_arg_wait_runner + self.CMD_STDIN = cmd_stdin_runner + self.CMD_ARG = cmd_arg_runner + self.CMD_URL_WAIT = cmd_url_wait_runner + + self.ALL = all def image_ref(self, repo): return posixpath.join(self.registry, repo) @@ -445,6 +413,8 @@ def run_echo_hello(self, repo: str): with timer(pull_cmd) as t: pull_elapsed = t + size = self.image_size(image_ref) + create_cmd = self.create_echo_hello_cmd(image_ref, container_name) print(create_cmd) @@ -452,7 +422,7 @@ def run_echo_hello(self, repo: str): with timer(create_cmd) as t: create_elapsed = t - run_cmd = self.task_start_cmd(container_name, iteration=False) + run_cmd = self.task_start_cmd(container_name, iteration=True) print(run_cmd) print("Running container %s ..." % container_name) @@ -461,11 +431,9 @@ def run_echo_hello(self, repo: str): if self.cleanup: self.clean_up(image_ref, container_name) - return pull_elapsed, create_elapsed, run_elapsed + return BenchResult(pull_elapsed, create_elapsed, run_elapsed, size) def run_cmd_arg(self, repo, runargs): - assert len(runargs.mount) == 0 - image_ref = self.image_ref(repo) container_name = repo.replace(":", "-") + random_chars() @@ -476,6 +444,8 @@ def run_cmd_arg(self, repo, runargs): with timer(pull_cmd) as t: pull_elapsed = t + size = self.image_size(image_ref) + create_cmd = self.create_cmd_arg_cmd(image_ref, container_name, runargs) print(create_cmd) @@ -483,7 +453,7 @@ def run_cmd_arg(self, repo, runargs): with timer(create_cmd) as t: create_elapsed = t - run_cmd = self.task_start_cmd(container_name, iteration=False) + run_cmd = self.task_start_cmd(container_name, iteration=True) print(run_cmd) with timer(run_cmd) as t: @@ -492,7 +462,7 @@ def run_cmd_arg(self, repo, runargs): if self.cleanup: self.clean_up(image_ref, container_name) - return pull_elapsed, create_elapsed, run_elapsed + return BenchResult(pull_elapsed, create_elapsed, run_elapsed, size) def run_cmd_arg_wait(self, repo, runargs): image_ref = self.image_ref(repo) @@ -505,6 +475,8 @@ def run_cmd_arg_wait(self, repo, runargs): with timer(pull_cmd) as t: pull_elapsed = t + size = self.image_size(image_ref) + create_cmd = self.create_cmd_arg_wait_cmd(image_ref, container_name, runargs) print(create_cmd) @@ -542,7 +514,7 @@ def run_cmd_arg_wait(self, repo, runargs): if self.cleanup: self.clean_up(image_ref, container_name) - return pull_elapsed, create_elapsed, run_elapsed + return BenchResult(pull_elapsed, create_elapsed, run_elapsed, size) def run_cmd_stdin(self, repo, runargs): image_ref = self.image_ref(repo) @@ -555,6 +527,8 @@ def run_cmd_stdin(self, repo, runargs): with timer(pull_cmd) as t: pull_elapsed = t + size = self.image_size(image_ref) + create_cmd = self.create_cmd_stdin_cmd(image_ref, container_name, runargs) print(create_cmd) @@ -590,7 +564,7 @@ def run_cmd_stdin(self, repo, runargs): if self.cleanup: self.clean_up(image_ref, container_name) - return pull_elapsed, create_elapsed, run_elapsed + return BenchResult(pull_elapsed, create_elapsed, run_elapsed, size) def run_cmd_url_wait(self, repo, runargs): image_ref = self.image_ref(repo) @@ -603,6 +577,8 @@ def run_cmd_url_wait(self, repo, runargs): with timer(pull_cmd) as t: pull_elapsed = t + size = self.image_size(image_ref) + create_cmd = self.create_cmd_url_wait_cmd(image_ref, container_id, runargs) print(create_cmd) @@ -634,25 +610,23 @@ def run_cmd_url_wait(self, repo, runargs): if self.cleanup: self.clean_up(image_ref, container_id) - return pull_elapsed, create_elapsed, run_elapsed + return BenchResult(pull_elapsed, create_elapsed, run_elapsed, size) def run(self, bench): repo = image_repo(bench.name) - if repo in BenchRunner.ECHO_HELLO: + if repo in self.ECHO_HELLO: return self.run_echo_hello(repo=bench.name) - elif repo in BenchRunner.CMD_ARG: - return self.run_cmd_arg(repo=bench.name, runargs=BenchRunner.CMD_ARG[repo]) - elif repo in BenchRunner.CMD_ARG_WAIT: + elif repo in self.CMD_ARG: + return self.run_cmd_arg(repo=bench.name, runargs=self.CMD_ARG[repo]) + elif repo in self.CMD_ARG_WAIT: return self.run_cmd_arg_wait( - repo=bench.name, runargs=BenchRunner.CMD_ARG_WAIT[repo] - ) - elif repo in BenchRunner.CMD_STDIN: - return self.run_cmd_stdin( - repo=bench.name, runargs=BenchRunner.CMD_STDIN[repo] + repo=bench.name, runargs=self.CMD_ARG_WAIT[repo] ) - elif repo in BenchRunner.CMD_URL_WAIT: + elif repo in self.CMD_STDIN: + return self.run_cmd_stdin(repo=bench.name, runargs=self.CMD_STDIN[repo]) + elif repo in self.CMD_URL_WAIT: return self.run_cmd_url_wait( - repo=bench.name, runargs=BenchRunner.CMD_URL_WAIT[repo] + repo=bench.name, runargs=self.CMD_URL_WAIT[repo] ) else: print("Unknown bench: " + repo) @@ -664,11 +638,32 @@ def pull_cmd(self, image_ref): f"nerdctl --snapshotter {self.snapshotter} pull {insecure_flag} {image_ref}" ) + def image_size(self, image_ref): + cmd = f"nerdctl images {str(image_ref)}" + print(cmd) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + out = p.communicate()[0].decode("utf-8") + return " ".join(out.split()[-2:]) + def create_echo_hello_cmd(self, image_ref, container_id): return f"nerdctl --snapshotter {self.snapshotter} create --net=host --name={container_id} {image_ref} -- echo hello" def create_cmd_arg_cmd(self, image_ref, container_id, runargs): - cmd = f"nerdctl --snapshotter {self.snapshotter} create --net=host --name={container_id} {image_ref} " + cmd = f"nerdctl --snapshotter {self.snapshotter} create --net=host " + if len(runargs.env) > 0: + env = " ".join(["--env %s=%s" % (k, v) for k, v in runargs.env.items()]) + cmd += f" {env} " + for a, b in runargs.mount: + a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) + a = tmp_copy(a) + cmd += f"--volume {a}:{b} " + if len(runargs.runtime) > 0: + cmd += f"--runtime {runargs.runtime} " + if len(runargs.shmSize) > 0: + cmd += f"--shm-size {runargs.shmSize} " + if len(runargs.workDir) > 0: + cmd += f"-w {runargs.workDir} " + cmd += f"--name={container_id} {image_ref} " return cmd + runargs.arg def create_cmd_arg_wait_cmd(self, image_ref, container_id, runargs): @@ -680,9 +675,15 @@ def create_cmd_arg_wait_cmd(self, image_ref, container_id, runargs): a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) a = tmp_copy(a) cmd += f"--volume {a}:{b} " - cmd += f"--name={container_id} {image_ref}" + cmd += f"--name={container_id} {image_ref} " + if len(runargs.runtime) > 0: + cmd += f"--runtime {runargs.runtime} " + if len(runargs.shmSize) > 0: + cmd += f"--shm-size {runargs.shmSize} " + if len(runargs.workDir) > 0: + cmd += f"-w {runargs.workDir} " if len(runargs.arg) > 0: - cmd += f" -- {runargs.arg} " + cmd += f"{runargs.arg} " return cmd @@ -692,9 +693,15 @@ def create_cmd_stdin_cmd(self, image_ref, container_id, runargs): a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) a = tmp_copy(a) cmd += f"--volume {a}:{b} " - cmd += f"--name={container_id} {image_ref}" + cmd += f"--name={container_id} {image_ref} " + if len(runargs.runtime) > 0: + cmd += f"--runtime {runargs.runtime} " + if len(runargs.shmSize) > 0: + cmd += f"--shm-size {runargs.shmSize} " + if len(runargs.workDir) > 0: + cmd += f"-w {runargs.workDir} " if runargs.stdin_sh: - cmd += f" -- {runargs.stdin_sh}" # e.g., sh -c + cmd += f"-- {runargs.stdin_sh}" # e.g., sh -c return cmd def create_cmd_url_wait_cmd(self, image_ref, container_id, runargs): @@ -706,9 +713,15 @@ def create_cmd_url_wait_cmd(self, image_ref, container_id, runargs): if len(runargs.env) > 0: env = " ".join([f"--env {k}={v}" for k, v in runargs.env.items()]) cmd += f" {env} " - cmd += f"--name={container_id} {image_ref}" + if len(runargs.runtime) > 0: + cmd += f"--runtime {runargs.runtime} " + if len(runargs.shmSize) > 0: + cmd += f"--shm-size {runargs.shmSize} " + if len(runargs.workDir) > 0: + cmd += f"-w {runargs.workDir} " + cmd += f"--name={container_id} {image_ref} " if len(runargs.arg) > 0: - cmd += f" -- {runargs.arg} " + cmd += f"{runargs.arg} " return cmd def task_start_cmd(self, container_id, iteration: bool): @@ -846,6 +859,13 @@ def main(): required=False, ) + parser.add_argument( + "--bench-config", + dest="bench_config", + required=False, + default="bench.yaml", + ) + parser.add_argument( "--out-format", dest="output_format", @@ -858,7 +878,7 @@ def main(): "--bench-times", dest="bench_times", type=int, - default= 1, + default=1, ) args = parser.parse_args() @@ -872,16 +892,29 @@ def main(): snapshotter = args.snapshotter cleanup = not args.no_cleanup insecure_registry = args.insecure_registry + bench_config = args.bench_config + + runner = BenchRunner( + docker=docker, + registry=registry, + registry2=registry2, + snapshotter=snapshotter, + cleanup=cleanup, + insecure_registry=insecure_registry, + bench_config=bench_config, + ) + + runner.load_bench_config() output_format = args.output_format bench_times = args.bench_times if all_supported_images: - benches.extend(BenchRunner.ALL.values()) + benches.extend(runner.ALL.values()) else: for i in images_list: try: - bench = copy.deepcopy(BenchRunner.ALL[image_repo(i)]) + bench = copy.deepcopy(runner.ALL[image_repo(i)]) tag = image_tag(i) if tag is not None: @@ -895,24 +928,20 @@ def main(): op = kvargs.pop("op", "run") f = open(outpath + "." + output_format, "w") - # run benchmarks - runner = BenchRunner( - docker=docker, - registry=registry, - registry2=registry2, - snapshotter=snapshotter, - cleanup=cleanup, - insecure_registry=insecure_registry, - ) - if output_format == "csv": - csv_headers = "timestamp,repo,bench,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s)" + csv_headers = "timestamp,repo,bench,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s),image_size" f.writelines(csv_headers + "\n") f.flush() for bench in benches: for _ in range(bench_times): - pull_elapsed, create_elapsed, run_elapsed = runner.operation(op, bench) + bench_result = runner.operation(op, bench) + pull_elapsed, create_elapsed, run_elapsed, size = ( + bench_result.pull_elapsed, + bench_result.create_elapsed, + bench_result.run_elapsed, + bench_result.size, + ) total_elapsed = f"{pull_elapsed + create_elapsed + run_elapsed: .6f}" timetamp = int(time.time() * 1000) @@ -929,10 +958,11 @@ def main(): "create_elapsed": create_elapsed, "run_elapsed": run_elapsed, "total_elapsed": total_elapsed, + "image_size": size, } line = json.dumps(row) elif output_format == "csv": - line = f"{timetamp},{bench.repo},{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed}" + line = f"{timetamp},{bench.repo},{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed},{size}" print(line) f.writelines(line + "\n") diff --git a/misc/ml/chat-bench/Dockerfile b/misc/ml/chat-bench/Dockerfile new file mode 100644 index 0000000..d60cd75 --- /dev/null +++ b/misc/ml/chat-bench/Dockerfile @@ -0,0 +1,11 @@ +FROM pytorch/pytorch:2.0.0-cuda11.7-cudnn8-devel + +ADD applications/Chat /application/chat +ADD transformers /application/transformers +WORKDIR /application + +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple +RUN cd /application/chat && pip install . +RUN cd /application/transformers && pip install . + +ENTRYPOINT [ "/application/chat/benchmarks/benchmark_gpt_dummy.sh" ] \ No newline at end of file diff --git a/misc/ml/chat-bench/build.sh b/misc/ml/chat-bench/build.sh new file mode 100755 index 0000000..b0480bf --- /dev/null +++ b/misc/ml/chat-bench/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +DIR=$(dirname "$0") + +# clone ColossalAI +git clone github.com/hpcaitech/ColossalAI.git + +# build chat-bench +sudo docker build -t ml_platform/chat-bench:2.0_cu117 ${DIR} \ No newline at end of file diff --git a/clojure/hello/.gitignore b/misc/mount/clojure/hello/.gitignore similarity index 100% rename from clojure/hello/.gitignore rename to misc/mount/clojure/hello/.gitignore diff --git a/clojure/hello/LICENSE b/misc/mount/clojure/hello/LICENSE similarity index 100% rename from clojure/hello/LICENSE rename to misc/mount/clojure/hello/LICENSE diff --git a/clojure/hello/README.md b/misc/mount/clojure/hello/README.md similarity index 100% rename from clojure/hello/README.md rename to misc/mount/clojure/hello/README.md diff --git a/clojure/hello/doc/intro.md b/misc/mount/clojure/hello/doc/intro.md similarity index 100% rename from clojure/hello/doc/intro.md rename to misc/mount/clojure/hello/doc/intro.md diff --git a/clojure/hello/project.clj b/misc/mount/clojure/hello/project.clj similarity index 100% rename from clojure/hello/project.clj rename to misc/mount/clojure/hello/project.clj diff --git a/clojure/hello/src/hello/core.clj b/misc/mount/clojure/hello/src/hello/core.clj similarity index 100% rename from clojure/hello/src/hello/core.clj rename to misc/mount/clojure/hello/src/hello/core.clj diff --git a/clojure/hello/test/hello/core_test.clj b/misc/mount/clojure/hello/test/hello/core_test.clj similarity index 100% rename from clojure/hello/test/hello/core_test.clj rename to misc/mount/clojure/hello/test/hello/core_test.clj diff --git a/gcc/main.c b/misc/mount/gcc/main.c similarity index 100% rename from gcc/main.c rename to misc/mount/gcc/main.c diff --git a/go/main.go b/misc/mount/go/main.go similarity index 100% rename from go/main.go rename to misc/mount/go/main.go diff --git a/iojs/index.js b/misc/mount/iojs/index.js similarity index 100% rename from iojs/index.js rename to misc/mount/iojs/index.js diff --git a/java/Main.java b/misc/mount/java/Main.java similarity index 100% rename from java/Main.java rename to misc/mount/java/Main.java diff --git a/mono/main.cs b/misc/mount/mono/main.cs similarity index 100% rename from mono/main.cs rename to misc/mount/mono/main.cs diff --git a/node/index.js b/misc/mount/node/index.js similarity index 100% rename from node/index.js rename to misc/mount/node/index.js diff --git a/thrift/hello.idl b/misc/mount/thrift/hello.idl similarity index 100% rename from thrift/hello.idl rename to misc/mount/thrift/hello.idl diff --git a/ml_image_list.txt b/ml_image_list.txt new file mode 100644 index 0000000..57ce593 --- /dev/null +++ b/ml_image_list.txt @@ -0,0 +1,4 @@ +tritonserver:23.02-py3 +pytorch:1.11_cu115 +pytorch-ddp:1.12.1 +tensorflow:2.4 diff --git a/ml_prepare.sh b/ml_prepare.sh new file mode 100755 index 0000000..8a66cc3 --- /dev/null +++ b/ml_prepare.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +function download_pytorch_benchmark() { + target=$(pwd)/misc/mount/pytorch + if [[ -d ${target} ]];then + echo "${target} already exists" + return + fi + git clone https://github.com/JunhongXu/pytorch-benchmark-volta.git $(pwd)/misc/mount/pytorch-benchmark-volta + rm -rf $(pwd)/misc/mount/pytorch-benchmark-volta/.git + mv $(pwd)/misc/mount/pytorch-benchmark-volta ${target} +} + +function download_tf_benchmark() { + target=$(pwd)/misc/mount/tf_cnn_benchmarks + if [[ -d ${target} ]];then + echo "${target} already exists" + return + fi + git clone https://github.com/tensorflow/benchmarks.git $(pwd)/misc/mount/benchmarks + mv $(pwd)/misc/mount/benchmarks/scripts/tf_cnn_benchmarks ${target} + rm -rf $(pwd)/misc/mount/benchmarks +} + +function download_model_repository() { + target=$(pwd)/misc/mount/model_repository + if [[ -d ${target} ]];then + echo "${target} already exists" + return + fi + mkdir -p ${target}/inception_graphdef/1 + wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \ + https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz + (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) + mv /tmp/inception_v3_2016_08_28_frozen.pb ${target}/inception_graphdef/1/model.graphdef + + mkdir -p ${target}/densenet_onnx/1 + wget -O ${target}/densenet_onnx/1/model.onnx \ + https://contentmamluswest001.blob.core.windows.net/content/14b2744cf8d6418c87ffddc3f3127242/9502630827244d60a1214f250e3bbca7/08aed7327d694b8dbaee2c97b8d0fcba/densenet121-1.2.onnx +} + +download_pytorch_benchmark +download_tf_benchmark +download_model_repository \ No newline at end of file diff --git a/run.sh b/run.sh index 0e13e52..0d3e2e1 100755 --- a/run.sh +++ b/run.sh @@ -16,6 +16,8 @@ RESULT_FILE=result.txt RESULT_CSV=result.csv NYDUSIFY_BIN=$(which nydusify) NYDUS_IMAGE_BIN=$(which nydus-image) +BENCH_CONFIG=bench.yaml +ACCESSED_LIST_DIR=accessed_list ######################################################### # Could alert value via arguments @@ -25,7 +27,7 @@ RESULT_DIR=data SOURCE_REGISTRY=docker.io/library TARGET_REGISTRY="" SKIP=false -IMAGES_PATH=hello_bench_image_list.txt +IMAGES_PATH=image_list.txt ######################################################### # Push OCI image to TARGET_REGISTRY @@ -62,18 +64,40 @@ function convert() { image=$1 + name=$(echo ${image} | awk -F: '{print $1}') + tag=$(echo ${image} | awk -F: '{print $2}') + if [[ "${tag}" == "" ]];then + tag=latest + fi + sudo nerdctl pull ${TARGET_REGISTRY}/${image} - echo "[INFO] Converting ${TARGET_REGISTRY}/${image} to ${TARGET_REGISTRY}/${image}:nydusv6 ..." + echo "[INFO] Converting ${TARGET_REGISTRY}/${image} to ${TARGET_REGISTRY}/${name}:${tag}-nydusv6 ..." echo "sudo $NYDUSIFY_BIN convert \ --fs-version 6 \ --nydus-image $NYDUS_IMAGE_BIN \ --source ${TARGET_REGISTRY}/${image} \ - --target ${TARGET_REGISTRY}/${image}:nydusv6" + --target ${TARGET_REGISTRY}/${name}:${tag}-nydusv6" sudo $NYDUSIFY_BIN convert \ --fs-version 6 \ --nydus-image $NYDUS_IMAGE_BIN \ --source ${TARGET_REGISTRY}/${image} \ - --target ${TARGET_REGISTRY}/${image}:nydusv6 + --target ${TARGET_REGISTRY}/${name}:${tag}-nydusv6 + + if [[ -f ${ACCESSED_LIST_DIR}/${image} ]];then + echo "[INFO] Converting ${TARGET_REGISTRY}/${image} to ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 ..." + echo "sudo $NYDUSIFY_BIN convert \ + --fs-version 6 \ + --nydus-image $NYDUS_IMAGE_BIN \ + --source ${TARGET_REGISTRY}/${image} \ + --target ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 \ + --prefetch-patterns < ${ACCESSED_LIST_DIR}/${image}.shuffed" + sudo $NYDUSIFY_BIN convert \ + --fs-version 6 \ + --nydus-image $NYDUS_IMAGE_BIN \ + --source ${TARGET_REGISTRY}/${image} \ + --target ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 \ + --prefetch-patterns < ${ACCESSED_LIST_DIR}/${image}.shuffed + fi } ######################################################### @@ -92,10 +116,12 @@ function stop_all_containers { else echo "Killing containers ${containers}" for C in ${containers}; do - sudo nerdctl kill "${C}" - sudo nerdctl stop "${C}" - sudo nerdctl rm "${C}" + sudo nerdctl kill "${C}" >/dev/null 2>&1 + sudo nerdctl stop "${C}" >/dev/null 2>&1 + sudo nerdctl rm -f "${C}" >/dev/null 2>&1 done + sudo nerdctl ps -a + sudo nerdctl images return 1 fi } @@ -112,34 +138,100 @@ function stop_all_containers { function run() { image=$1 + name=$(echo ${image} | awk -F: '{print $1}') + tag=$(echo ${image} | awk -F: '{print $2}') + if [[ "${tag}" == "" ]];then + tag=latest + fi + stop_all_containers sudo nerdctl ps -a | awk 'NR>1 {print $1}' | xargs sudo nerdctl rm >/dev/null 2>&1 - sudo nerdctl container prune -f - sudo nerdctl image prune -f --all - sudo systemctl restart nydus-snapshotter + sudo nerdctl container prune -f >/dev/null 2>&1 + sudo nerdctl image prune --all -f >/dev/null 2>&1 + sudo nerdctl system prune --volumes --all -f >/dev/null 2>&1 + sudo ctr leases ls | awk 'NR>1 {print $1}' | xargs sudo ctr leases rm --sync >/dev/null 2>&1 + sudo ctr images ls | awk 'NR>1 {print $1}' | xargs sudo ctr images rm --sync >/dev/null 2>&1 + sudo ctr content prune references >/dev/null 2>&1 sleep 1 echo "[INFO] Run hello bench in ${image} ..." sudo nerdctl --snapshotter overlayfs rmi -f ${TARGET_REGISTRY}/${image} >/dev/null 2>&1 - result=$(sudo ./hello.py --engine nerdctl --snapshotter overlayfs --op run \ + result=$(sudo ./hello.py --bench-config=${BENCH_CONFIG} --engine nerdctl --snapshotter overlayfs --op run \ --registry=${TARGET_REGISTRY} \ --images ${image} | - grep "repo") + grep "repo" | grep "bench" | grep "timestamp") echo ${result} echo ${result} >>${RESULT_DIR}/${RESULT_FILE}.${CURRENT_ROUND} echo "[INFO] Remove image ${TARGET_REGISTRY}/${image} ..." sudo nerdctl --snapshotter overlayfs rmi -f ${TARGET_REGISTRY}/${image} >/dev/null 2>&1 +} + +function run_nydus() { + image=$1 + + name=$(echo ${image} | awk -F: '{print $1}') + tag=$(echo ${image} | awk -F: '{print $2}') + if [[ "${tag}" == "" ]];then + tag=latest + fi + + stop_all_containers + sudo nerdctl ps -a | awk 'NR>1 {print $1}' | xargs sudo nerdctl rm >/dev/null 2>&1 + sudo nerdctl container prune -f >/dev/null 2>&1 + sudo nerdctl image prune --all -f >/dev/null 2>&1 + sudo nerdctl system prune --volumes --all -f >/dev/null 2>&1 + sudo ctr leases ls | awk 'NR>1 {print $1}' | xargs sudo ctr leases rm --sync >/dev/null 2>&1 + sudo ctr images ls | awk 'NR>1 {print $1}' | xargs sudo ctr images rm --sync >/dev/null 2>&1 + sudo ctr content prune references >/dev/null 2>&1 + sleep 1 + sudo lsof -nP +L1 | grep containerd | grep "(deleted)" | awk '{print $2}' | xargs sudo kill + sleep 1 + sudo rm -rf /var/lib/containerd/nydus/cache - echo "[INFO] Run hello bench in ${image}:nydusv6 ..." - sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${image}:nydusv6 >/dev/null 2>&1 - result=$(sudo ./hello.py --engine nerdctl --snapshotter nydus --op run \ + echo "[INFO] Run hello bench in ${name}:${tag}-nydusv6 ..." + sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${name}:${tag}-nydusv6 >/dev/null 2>&1 + result=$(sudo ./hello.py --bench-config=${BENCH_CONFIG} --engine nerdctl --snapshotter nydus --op run \ --registry=${TARGET_REGISTRY} \ - --images ${image}:nydusv6 | - grep "repo") + --images ${name}:${tag}-nydusv6 | + grep "repo" | grep "bench" | grep "timestamp") echo ${result} - echo ${result} >>${RESULT_DIR}/${RESULT_FILE}.${CURRENT_ROUND} - echo "[INFO] Remove image ${TARGET_REGISTRY}/${image}:nydusv6 ..." - sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${image}:nydusv6 >/dev/null 2>&1 + echo ${result} >>${RESULT_DIR}/${RESULT_FILE}.nydus.${CURRENT_ROUND} + echo "[INFO] Remove image ${TARGET_REGISTRY}/${name}:${tag}-nydusv6 ..." + sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${name}:${tag}-nydusv6 >/dev/null 2>&1 +} + +function run_optimized_nydus() { + image=$1 + + name=$(echo ${image} | awk -F: '{print $1}') + tag=$(echo ${image} | awk -F: '{print $2}') + if [[ "${tag}" == "" ]];then + tag=latest + fi + + stop_all_containers + sudo nerdctl ps -a | awk 'NR>1 {print $1}' | xargs sudo nerdctl rm >/dev/null 2>&1 + sudo nerdctl container prune -f >/dev/null 2>&1 + sudo nerdctl image prune --all -f >/dev/null 2>&1 + sudo nerdctl system prune --volumes --all -f >/dev/null 2>&1 + sudo ctr leases ls | awk 'NR>1 {print $1}' | xargs sudo ctr leases rm --sync >/dev/null 2>&1 + sudo ctr images ls | awk 'NR>1 {print $1}' | xargs sudo ctr images rm --sync >/dev/null 2>&1 + sudo ctr content prune references >/dev/null 2>&1 + sleep 1 + sudo lsof -nP +L1 | grep containerd | grep "(deleted)" | awk '{print $2}' | xargs sudo kill + sleep 1 + sudo rm -rf /var/lib/containerd/nydus/cache + + echo "[INFO] Run hello bench in ${name}:${tag}-optimized-nydusv6 ..." + sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 >/dev/null 2>&1 + result=$(sudo ./hello.py --bench-config=${BENCH_CONFIG} --engine nerdctl --snapshotter nydus --op run \ + --registry=${TARGET_REGISTRY} \ + --images ${name}:${tag}-optimized-nydusv6 | + grep "repo" | grep "bench" | grep "timestamp") + echo ${result} + echo ${result} >>${RESULT_DIR}/${RESULT_FILE}.optimized.nydus.${CURRENT_ROUND} + echo "[INFO] Remove image ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 ..." + sudo nerdctl --snapshotter nydus rmi -f ${TARGET_REGISTRY}/${name}:${tag}-optimized-nydusv6 >/dev/null 2>&1 } ######################################################### @@ -219,6 +311,7 @@ function usage() { echo "Usage:" echo -e "run.sh -o OPERATION -s SOURCE_REGISTRY -t TARGET_REGISTRY [other options] [-o operation] \tavailable options are [ push convert run all draw ] +[-c config] \tbench config file (only available for \"run\" operation) [-i images] \timages list [-p images path] \tfile path that contains images list (line by line) [-s source registry] \tsource registry for pulling image @@ -243,7 +336,7 @@ if [ $# -eq 0 ]; then usage fi -while getopts o:i:p:s:t:r:d:kh OPT; do +while getopts o:c:i:p:s:t:r:d:k:h OPT; do case $OPT in o) operation=${OPTARG} @@ -252,6 +345,9 @@ while getopts o:i:p:s:t:r:d:kh OPT; do exit fi + ;; + c) + BENCH_CONFIG=${OPTARG} ;; i) getopts_extra "$@" @@ -327,6 +423,8 @@ run) CURRENT_ROUND=${i} if [ ! "${SKIP}" == "true" ]; then echo "" >${RESULT_DIR}/${RESULT_FILE}.${CURRENT_ROUND} + echo "" >${RESULT_DIR}/${RESULT_FILE}.nydus.${CURRENT_ROUND} + echo "" >${RESULT_DIR}/${RESULT_FILE}.optimized.nydus.${CURRENT_ROUND} fi for image in "${IMAGES[@]}"; do @@ -345,6 +443,38 @@ run) fi run ${image} done + for image in "${IMAGES[@]}"; do + if [ "${SKIP}" == "true" ]; then + skip=false + for i in $(cat ${RESULT_DIR}/${RESULT_FILE}.nydus.${CURRENT_ROUND}); do + if [[ "${i}" =~ "${image}" ]]; then + echo "Skip image ${image}." + skip=true + break + fi + done + if [ "${skip}" == "true" ]; then + continue + fi + fi + run_nydus ${image} + done + for image in "${IMAGES[@]}"; do + if [ "${SKIP}" == "true" ]; then + skip=false + for i in $(cat ${RESULT_DIR}/${RESULT_FILE}.optimized.nydus.${CURRENT_ROUND}); do + if [[ "${i}" =~ "${image}" ]]; then + echo "Skip image ${image}." + skip=true + break + fi + done + if [ "${skip}" == "true" ]; then + continue + fi + fi + run_optimized_nydus ${image} + done done ;; all) diff --git a/template/Dockerfile b/template/Dockerfile new file mode 100644 index 0000000..c270549 --- /dev/null +++ b/template/Dockerfile @@ -0,0 +1,11 @@ +FROM bash:latest + +LABEL maintainer="sctb512@gmail.com" + +ADD entrypoint.sh /entrypoint.sh +ADD file_list_path.txt /file_list_path.txt +ADD file_list_path_shuffed.txt /file_list_path_shuffed.txt + +COPY REPLACE_ME REPLACE_ME + +ENTRYPOINT [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/template/bench.yaml b/template/bench.yaml new file mode 100644 index 0000000..4b26e43 --- /dev/null +++ b/template/bench.yaml @@ -0,0 +1,2 @@ +CMD_ARG_WAIT: + REPLACE_ME \ No newline at end of file diff --git a/template/entrypoint.sh b/template/entrypoint.sh new file mode 100755 index 0000000..e2d623c --- /dev/null +++ b/template/entrypoint.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +default_type=seq +default_file_list_path=/file_list_path.txt +if [[ $# -eq 0 ]]; then + type=${default_type} + path=${default_file_list_path} +else + if [[ $# -eq 2 ]]; then + if [[ $1 != seq && $1 != rand ]]; then + echo "unsupported type: $1, available type: seq rand" + exit 1 + fi + type=$1 + path=$2 + else + echo "Usage: $0 READ_TYPE(seq rand) FILE_LIST_PATH" + exit 1 + fi +fi + +echo "read type: $type" +echo "file list path: $path" + +if [[ $type == seq ]]; then + files=($(cat ${path} | tr "\n" " ")) + files_number=${#files[@]} + echo "read file number: $files_number" + + for file in "${files[@]}"; do + file_size=$(stat -c%s "${file}") + echo "file: ${file} size: ${file_size}" + cat ${file} >/dev/null + done +elif [[ $type == rand ]]; then + rand_path=${path}.rand + shuf ${path} -o ${path}.rand + + files=($(cat ${rand_path} | tr "\n" " ")) + files_number=${#files[@]} + echo "read file number: $files_number" + + for file in "${files[@]}"; do + file_size=$(stat -c%s "${file}") + echo "file: ${file} size: ${file_size}" + cat ${file} >/dev/null + done +fi + +echo "Read file list done."