Skip to content

Commit f763f3f

Browse files
ya-developerya-developer
authored
Merge pull request #128 from ProblemSetters/performance-optimizations
Various performance optimizations
2 parents abb4415 + 8ed2fb0 commit f763f3f

35 files changed

Lines changed: 190 additions & 135 deletions

‎.github/workflows/ecr/hr/ubuntu-sshd-openvpn/22.04/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ COPY rootfs/ /
44

55
ENV ECR=134148934511.dkr.ecr.us-east-1.amazonaws.com/hr/ubuntu-sshd-openvpn:22.04
66

7-
RUN apt-get \
7+
RUN apt \
88
update \
99
&& \
1010
DEBIAN_FRONTEND=noninteractive \
11-
apt-get \
11+
apt \
1212
install \
1313
-y \
1414
--no-install-recommends \

‎.github/workflows/ecr/hr/ubuntu-sshd-wireguard/22.04/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ COPY rootfs/ /
44

55
ENV ECR=134148934511.dkr.ecr.us-east-1.amazonaws.com/hr/ubuntu-sshd-wireguard:22.04
66

7-
RUN apt-get \
7+
RUN apt \
88
update \
99
&& \
1010
DEBIAN_FRONTEND=noninteractive \
11-
apt-get \
11+
apt \
1212
install \
1313
-y \
1414
--no-install-recommends \

‎.github/workflows/ecr/hr/ubuntu-sshd/22.04/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ COPY rootfs/ /
44

55
ENV ECR=134148934511.dkr.ecr.us-east-1.amazonaws.com/hr/ubuntu-sshd:22.04
66

7-
RUN apt-get \
7+
RUN apt \
88
update \
99
&& \
1010
DEBIAN_FRONTEND=noninteractive \
11-
apt-get \
11+
apt \
1212
install \
1313
-y \
1414
--no-install-recommends \

‎Makefile.blackbox‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ debug: ;@:
5050
all: uninstall install setup solve check
5151

5252
install:
53-
docker login --username AWS --password-stdin 134148934511.dkr.ecr.us-east-1.amazonaws.com < <(docker run -it --env="AWS_DEFAULT_REGION=$(BLACKBOX_AWS_DEFAULT_REGION)" --env="AWS_ACCESS_KEY_ID=$(BLACKBOX_AWS_ACCESS_KEY_ID)" --env="AWS_SECRET_ACCESS_KEY=$(BLACKBOX_AWS_SECRET_ACCESS_KEY)" --rm amazon/aws-cli ecr get-login-password)
53+
docker login --username AWS --password-stdin 134148934511.dkr.ecr.us-east-1.amazonaws.com < <(docker run -it --env="AWS_DEFAULT_REGION=$(BLACKBOX_AWS_DEFAULT_REGION)" --env="AWS_ACCESS_KEY_ID=$(BLACKBOX_AWS_ACCESS_KEY_ID)" --env="AWS_SECRET_ACCESS_KEY=$(BLACKBOX_AWS_SECRET_ACCESS_KEY)" --rm amazon/aws-cli:2.11.25 ecr get-login-password)
5454
cat <(echo FROM 134148934511.dkr.ecr.us-east-1.amazonaws.com/hr/blackbox_2204:abstract) <(echo RUN groupadd docker) | docker build --build-arg="BLACKBOX_USER=$(BLACKBOX_USER)" --build-arg="BLACKBOX_USER_NAME=$(BLACKBOX_USER_NAME)" --build-arg="BLACKBOX_USER_HOME=$(BLACKBOX_USER_HOME)" --build-arg="BLACKBOX_VERSION=$(BLACKBOX_VERSION)" --network="host" --progress="plain" --tag="blackbox:$(BLACKBOX_HOST)" -
5555
docker run -di --group-add docker --hostname="blackbox" --name="$(BLACKBOX_HOST)" --privileged --cgroupns=host $(if $(filter debug,$(MAKECMDGOALS)),--volume="$(BLACKBOX_DIR):/blackbox:ro") --volume="$(call blackbox/pwd)/setup.sh:$(BLACKBOX_USER_HOME)/setup.sh:ro" --volume="$(call blackbox/pwd)/check.sh:$(BLACKBOX_USER_HOME)/check.sh:ro" --volume="$(call blackbox/pwd)/solve.sh:$(BLACKBOX_USER_HOME)/solve.sh:ro" blackbox:$(BLACKBOX_HOST) >/dev/null
5656

‎blackbox‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ blackbox() {
6666
export BLACKBOX_HEAP_DIR="/tmp/${3}"
6767
export BLACKBOX_STORAGE_DIR="/var/save/${3}"
6868
export BLACKBOX_BUILD_LOG="${BLACKBOX_STORAGE_DIR}/build.log"
69-
export BLACKBOX_BUILD_STREAM_DISPLAY_LINES=255
69+
export BLACKBOX_BUILD_STREAM_DISPLAY_LINES=23456
7070

7171
# shellcheck disable=SC2034
7272
typeset -gx BLACKBOX_SCENARIO_PATH=$(realpath "$0")
@@ -230,20 +230,12 @@ blackbox() {
230230
(
231231
if [ -n "$header" ]; then
232232
printf "\n"
233-
234-
if [[ "$BLACKBOX_FLAG__DEBUG_MODE" == "yes" ]]; then
235-
printf "\033[0;37m// %s%s%s\033[0m\n" "${entrypoint[0]}" "${args[*]}" "${opts[*]}"
236-
fi
237-
233+
printf "\033[0;37m// @%s~%s %s%s%s\033[0m\n" "$(date -u --iso-8601=seconds)" "$SECONDS" "${entrypoint[0]}" "${args[*]}" "${opts[*]}"
238234
printf "\033[0;34m[%s]\033[0m\n" "${header^^}"
239235
printf "\033[0;37m%s\033[0m\n" "$(printenv | sort | awk 'ORS=" "')"
240236
printf "\033[0;37m--\033[0m\n"
241237
else
242-
if [[ "$BLACKBOX_FLAG__DEBUG_MODE" != "yes" ]]; then
243-
return
244-
fi
245-
246-
printf "\033[0;37m// %s%s%s {...}\033[0m\n" "${entrypoint[0]}" "${args[*]}" "${opts[*]}"
238+
printf "\033[0;37m// @%s~%s %s%s%s {...}\033[0m\n" "$(date -u --iso-8601=seconds)" "$SECONDS" "${entrypoint[0]}" "${args[*]}" "${opts[*]}"
247239
fi
248240
) | if ( tput setaf &>/dev/null ); then cat; else sed "s/\x1b\[[0-9;]*m//g"; fi | if [[ "$BLACKBOX_FLAG__DEBUG_MODE" == "yes" ]]; then cat >&2; else cat; fi
249241
}
@@ -263,6 +255,8 @@ blackbox() {
263255
typeset module_name=$1
264256
typeset -a module_args=("${@:2}")
265257

258+
typeset -gx SECONDS=0
259+
266260
{
267261
shopt -os allexport
268262
. /etc/environment
@@ -274,8 +268,8 @@ blackbox() {
274268
}
275269

276270
if [ ! -e "$BLACKBOX_DIR" ]; then
277-
curl -H "Cache-Control: no-cache, no-store" -H "Pragma: no-cache" "https://codeload.github.com/ProblemSetters/devops-blackbox/zip/refs/heads/${BLACKBOX_VERSION}" --output /tmp/blackbox && {
278-
unzip /tmp/blackbox -d /tmp && {
271+
wget -S --output-document="/tmp/blackbox.zip" --no-verbose "https://codeload.github.com/ProblemSetters/devops-blackbox/zip/refs/heads/${BLACKBOX_VERSION}" && {
272+
unzip /tmp/blackbox.zip -x "devops-blackbox-2204/.github/*" "devops-blackbox-2204/docs/*" "devops-blackbox-2204/.gitignore" "devops-blackbox-2204/Makefile.*" "devops-blackbox-2204/README.md" "devops-blackbox-2204/Vagrantfile" -d /tmp && {
279273
mv "/tmp/devops-blackbox-${BLACKBOX_VERSION}" "$BLACKBOX_DIR"
280274
rm -rf /tmp/*blackbox*
281275
}

‎framework/inventory/ansible.inventory‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ blackbox.framework.inventory.ansible() {
3131
return
3232
fi
3333

34-
apt-get update
34+
if [ -z "$(find /var/lib/apt/lists -newermt "5 minutes ago")" ]; then
35+
apt update
36+
fi
3537

36-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends software-properties-common && {
38+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends software-properties-common && {
3739
apt-add-repository -y ppa:ansible/ansible
38-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$package_name" "${with_packages[@]}"
40+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends "$package_name" "${with_packages[@]}"
3941
}
4042
}

‎framework/inventory/awscli.inventory‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,25 @@ blackbox.framework.inventory.awscli() {
2323
blackbox.framework.trace "${FUNCNAME[0]}" "$*" <<<""
2424
}
2525

26-
apt-get update
26+
tee -a /home/ubuntu/.bashrc <<<"complete -C /usr/local/aws-cli/v2/current/bin/aws_completer aws"
2727

28-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends software-properties-common gpg-agent wget less && {
29-
wget -S -O /tmp/awscli.zip --no-verbose https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip && {
28+
tee -a /etc/environment < <(printf "AWS_DEFAULT_REGION=%s\n" "eu-central-1")
29+
tee -a /etc/environment < <(printf "AWS_ACCESS_KEY_ID=A%sIA%s%s%s\n" "$(head -c 1 <(tr -dc SK </dev/urandom))" "$(head -c 1 <(tr -dc IJ </dev/urandom))" "$(head -c 14 <(tr -dc A-Z2-7 </dev/urandom))" "$(head -c 1 <(tr -dc AQ </dev/urandom))")
30+
tee -a /etc/environment < <(printf "AWS_SECRET_ACCESS_KEY=%s\n" "$(head -c 40 <(tr -dc A-Za-z0-9+/ </dev/urandom))")
31+
32+
if [[ -x "/usr/local/aws-cli/v2/current/bin/aws" ]]; then
33+
return
34+
fi
35+
36+
if [ -z "$(find /var/lib/apt/lists -newermt "5 minutes ago")" ]; then
37+
apt update
38+
fi
39+
40+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends software-properties-common gpg-agent wget less && {
41+
wget -S --output-document="/tmp/awscli.zip" --no-verbose https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.11.25.zip && {
3042
rm -rf /usr/local/aws-cli/
3143

32-
unzip -q /tmp/awscli.zip -d /tmp && {
44+
unzip /tmp/awscli.zip -d /tmp && {
3345
/tmp/aws/install
3446
rm -rf /tmp/aws*
3547
}
@@ -38,14 +50,6 @@ blackbox.framework.inventory.awscli() {
3850
cp /blackbox/framework/inventory/awscli/rootfs/usr/local/bin/aws /usr/local/bin/ && {
3951
chmod +x /usr/local/bin/aws
4052
}
41-
42-
rm /usr/local/bin/aws_completer && {
43-
tee -a /home/ubuntu/.bashrc <<<"complete -C /usr/local/aws-cli/v2/current/bin/aws_completer aws"
44-
}
45-
46-
tee -a /etc/environment < <(printf "AWS_DEFAULT_REGION=%s\n" "eu-central-1")
47-
tee -a /etc/environment < <(printf "AWS_ACCESS_KEY_ID=A%sIA%s%s%s\n" "$(head -c 1 <(tr -dc SK </dev/urandom))" "$(head -c 1 <(tr -dc IJ </dev/urandom))" "$(head -c 14 <(tr -dc A-Z2-7 </dev/urandom))" "$(head -c 1 <(tr -dc AQ </dev/urandom))")
48-
tee -a /etc/environment < <(printf "AWS_SECRET_ACCESS_KEY=%s\n" "$(head -c 40 <(tr -dc A-Za-z0-9+/ </dev/urandom))")
4953
}
5054
}
5155
}

‎framework/inventory/boto3.inventory‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,11 @@ blackbox.framework.inventory.boto3() {
3131
return
3232
fi
3333

34-
apt-get update
34+
if [ -z "$(find /var/lib/apt/lists -newermt "5 minutes ago")" ]; then
35+
apt update
36+
fi
3537

36-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-pip && {
38+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends python3-pip && {
3739
local requirements="boto3"
3840

3941
for requirement in ${requirements[*]}; do
@@ -47,7 +49,7 @@ blackbox.framework.inventory.boto3() {
4749
return
4850
fi
4951

50-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "${with_packages[@]}"
52+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends "${with_packages[@]}"
5153
fi
5254
}
5355
}

‎framework/inventory/chef-workstation.inventory‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ blackbox.framework.inventory.chef-workstation() {
3131
return
3232
fi
3333

34-
apt-get update
34+
if [ -z "$(find /var/lib/apt/lists -newermt "5 minutes ago")" ]; then
35+
apt update
36+
fi
3537

36-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends software-properties-common gpg-agent wget && {
37-
wget --output-document="/tmp/chef.deb" https://packages.chef.io/files/stable/chef-workstation/22.6.973/ubuntu/18.04/chef-workstation_22.6.973-1_amd64.deb && {
38-
DEBIAN_FRONTEND=noninteractive apt-get install -y /tmp/chef.deb "${with_packages[@]}" && {
38+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends software-properties-common gpg-agent wget && {
39+
wget -S --output-document="/tmp/chef.deb" --no-verbose https://packages.chef.io/files/stable/chef-workstation/22.6.973/ubuntu/18.04/chef-workstation_22.6.973-1_amd64.deb && {
40+
DEBIAN_FRONTEND=noninteractive apt install -y /tmp/chef.deb "${with_packages[@]}" && {
3941
rm -f /tmp/chef.deb
4042
}
4143
}

‎framework/inventory/chef.inventory‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ blackbox.framework.inventory.chef() {
3131
return
3232
fi
3333

34-
apt-get update
34+
if [ -z "$(find /var/lib/apt/lists -newermt "5 minutes ago")" ]; then
35+
apt update
36+
fi
3537

36-
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends software-properties-common gpg-agent wget && {
37-
wget --output-document="/tmp/chef.deb" https://packages.chef.io/files/stable/chef/18.2.7/ubuntu/22.04/chef_18.2.7-1_amd64.deb && {
38-
DEBIAN_FRONTEND=noninteractive apt-get install -y /tmp/chef.deb "${with_packages[@]}" && {
38+
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends software-properties-common gpg-agent wget && {
39+
wget -S --output-document="/tmp/chef.deb" --no-verbose https://packages.chef.io/files/stable/chef/18.2.7/ubuntu/22.04/chef_18.2.7-1_amd64.deb && {
40+
DEBIAN_FRONTEND=noninteractive apt install -y /tmp/chef.deb "${with_packages[@]}" && {
3941
rm -f /tmp/chef.deb
4042
}
4143
}

0 commit comments

Comments
 (0)