diff --git a/generate_image.sh b/generate_image.sh new file mode 100755 index 0000000..1aca3c2 --- /dev/null +++ b/generate_image.sh @@ -0,0 +1,260 @@ +#!/bin/bash + +tmpdir=tmp +registry=docker.io/library + +image_list=images.txt +target_image_list=generated_image_list.txt +target_bench_yaml=generated_bench.yaml + +function large_number_small_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + file_number=(128 256 512 1024 2048 4096 8192 16384 32768 65536) + file_size=(1 2 4 8 16 32 64 128) + layer_number=(1 2 4 8 16 32 64) + for number in ${file_number[@]}; do + for size in ${file_size[@]}; do + for layer in ${layer_number[@]}; do + dir=${tmpdir}/dir_number${number}_size${size}_layer${layer} + mkdir -p ${dir} + + kb=$((${number} * ${size})) + kbplayer=$((${kb} / ${layer})) + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, size: ${size}, layer: ${layer}, total ${kb}KB, per layer: ${kbplayer}KB" + + image=${registry}/nydus-small-file-test-l${layer}:n${number}-s${size} + for i in $(cat ${image_list}); do + if [[ "${i}" =~ "${image}" ]]; then + echo "Skip image ${image}." + continue + fi + done + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1K count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function small_number_large_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + file_number=(1 2 4 8) + file_size=(1 2 4 8) + layer_number=(1) + for number in ${file_number[@]}; do + for size in ${file_size[@]}; do + for layer in $(seq ${number}); do + if [[ $((${number} % ${layer})) != 0 ]]; then + echo "skip number: ${number}, layer: ${layer}" + continue + fi + dir=${tmpdir}/dir_number${number}_size${size}_layer${layer} + mkdir -p ${dir} + + gb=$((${number} * ${size})) + kb=$((${number} * ${size} * ${gb} * 1024 * 1024)) + kbplayer=$((${kb} / ${layer})) + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, size: ${size}, layer: ${layer}, total ${kb}KB, per layer: ${kbplayer}KB" + + image=${registry}/nydus-small-file-test-l${layer}:n${number}-s${size} + for i in $(cat ${image_list}); do + if [[ "${i}" =~ "${image}" ]]; then + echo "Skip image ${image}." + continue + fi + done + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1G count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function large_number_random_files() { + if [ -d ${tmpdir} ]; then + sudo rm -rf ${tmpdir} + fi + mkdir ${tmpdir} + layer_number=(1 2 4 8 16 32 64) + for layer in ${layer_number[@]}; do + for i in $(seq 5); do + number=$(((RANDOM % 65536) + 100)) + for j in $(seq 5); do + dir=${tmpdir}/dir_number${i}_size${j}_layer${layer} + mkdir -p ${dir} + + fileplayer=$((${number} / ${layer})) + echo "number: ${number}, layer: ${layer}, fileplayer: ${fileplayer}" + + image=${registry}/nydus-random-file-test-l${layer}:n${i}-s${j} + for i in $(cat ${image_list}); do + if [[ "${i}" =~ "${image}" ]]; then + echo "Skip image ${image}." + continue + fi + done + + cp template/Dockerfile ${dir} + echo "" >${dir}/file_list_path.txt + + for l in $(seq ${layer}); do + layer_dir=${dir}/layer_${l} + mkdir -p ${layer_dir} + for f in $(seq ${fileplayer}); do + size=$(((RANDOM % 128) + 1)) + file_name=${layer_dir}/file_${f} + dd if=/dev/urandom of=${file_name} bs=1K count=${size} conv=notrunc >/dev/null 2>&1 + echo "/tmp/layer_${l}/file_${f}" >>${dir}/file_list_path.txt + done + sed -i "s/^COPY REPLACE_ME.*/COPY layer_${l} \/tmp\/layer_${l}\nCOPY REPLACE_ME REPLACE_ME/g" ${dir}/Dockerfile + done + + sed -i "/^$/d" ${dir}/file_list_path.txt + shuf ${dir}/file_list_path.txt -o ${dir}/file_list_path_shuffed.txt + + sed -i "/^COPY REPLACE_ME.*/d" ${dir}/Dockerfile + cp template/entrypoint.sh ${dir} + + sudo docker image load -i bash.latest + sudo docker build -t ${image} ${dir} + sudo docker push ${image} + echo ${image} >>${image_list} + rm -rf ${dir} + sudo docker rmi -f ${image} >/dev/null 2>&1 + sudo docker system prune -a -f >/dev/null 2>&1 + done + done + done +} + +function generate_bench_yaml() { + echo "" >${target_image_list} + cp template/bench.yaml ${target_bench_yaml} + images=($(cat ${image_list} | tr "\n" " ")) + for image in ${images[@]}; do + repo=$(echo ${image} | awk -F\/ '{print $(NF-1)}') + image_name=$(echo ${image} | awk -F\/ '{print $(NF)}') + + echo ${image_name} >>${target_image_list} + + name=$(echo ${image_name} | awk -F: '{print $1}') + if [[ "${tag}" == "" ]]; then + tag=latest + fi + sed -i "s/^ REPLACE_ME/- bench_args:\n wait_line: 'Read file list done'\n category: test\n image: ${name}\n repo: ${repo}\n REPLACE_ME/g" ${target_bench_yaml} + done + sed -i "/REPLACE_ME/d" ${target_bench_yaml} +} + +######################################################### +# Usage information +# Globals: +# None +# Arguments: +# None +# Returns: +# None +######################################################### +function usage() { + echo "Usage:" + echo -e "$0 [-t TARGET_REGISTRY] [-p TARGET_IMAGE_LIST_PATH] +[-t target registry] \target registry for pushing image +[-p target image list path] \tfile path that contains images list (line by line) +[-b target bench yaml path] \tfile path for bench configuration +" + exit -1 +} + +while getopts p:t:b:h OPT; do + case $OPT in + t) + registry=${OPTARG} + ;; + p) + target_image_list=${OPTARG} + ;; + b) + target_bench_yaml=${OPTARG} + ;; + *) + usage + ;; + esac +done +shift $((OPTIND - 1)) + +echo "target registry: ${registry}" +echo "target image list: ${target_image_list}" +echo "target bench yaml: ${target_bench_yaml}" + +if [[ ! -f ${image_list} ]]; then + touch ${image_list} +fi +sudo docker pull bash:latest +sudo docker image save bash:latest -o bash.latest + +large_number_small_files +small_number_large_files +large_number_random_files + +generate_bench_yaml diff --git a/template/Dockerfile b/template/Dockerfile new file mode 100644 index 0000000..c270549 --- /dev/null +++ b/template/Dockerfile @@ -0,0 +1,11 @@ +FROM bash:latest + +LABEL maintainer="sctb512@gmail.com" + +ADD entrypoint.sh /entrypoint.sh +ADD file_list_path.txt /file_list_path.txt +ADD file_list_path_shuffed.txt /file_list_path_shuffed.txt + +COPY REPLACE_ME REPLACE_ME + +ENTRYPOINT [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/template/bench.yaml b/template/bench.yaml new file mode 100644 index 0000000..4b26e43 --- /dev/null +++ b/template/bench.yaml @@ -0,0 +1,2 @@ +CMD_ARG_WAIT: + REPLACE_ME \ No newline at end of file diff --git a/template/entrypoint.sh b/template/entrypoint.sh new file mode 100755 index 0000000..e2d623c --- /dev/null +++ b/template/entrypoint.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +default_type=seq +default_file_list_path=/file_list_path.txt +if [[ $# -eq 0 ]]; then + type=${default_type} + path=${default_file_list_path} +else + if [[ $# -eq 2 ]]; then + if [[ $1 != seq && $1 != rand ]]; then + echo "unsupported type: $1, available type: seq rand" + exit 1 + fi + type=$1 + path=$2 + else + echo "Usage: $0 READ_TYPE(seq rand) FILE_LIST_PATH" + exit 1 + fi +fi + +echo "read type: $type" +echo "file list path: $path" + +if [[ $type == seq ]]; then + files=($(cat ${path} | tr "\n" " ")) + files_number=${#files[@]} + echo "read file number: $files_number" + + for file in "${files[@]}"; do + file_size=$(stat -c%s "${file}") + echo "file: ${file} size: ${file_size}" + cat ${file} >/dev/null + done +elif [[ $type == rand ]]; then + rand_path=${path}.rand + shuf ${path} -o ${path}.rand + + files=($(cat ${rand_path} | tr "\n" " ")) + files_number=${#files[@]} + echo "read file number: $files_number" + + for file in "${files[@]}"; do + file_size=$(stat -c%s "${file}") + echo "file: ${file} size: ${file_size}" + cat ${file} >/dev/null + done +fi + +echo "Read file list done."