graphregistry/run.sh at master · epflgraph/graphregistry · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/usr/bin/env bash
set -Eeuo pipefail
IFS=$'\n\t'

# -----------------------------
# Config
# -----------------------------
SEP=$'\n\n###########################################################################################\n\n'

# Memory limit for each step (KiB). 4 GiB = 4*1024*1024 KiB.
MEM_LIMIT_KIB=$((4*1024*1024))

# If you want a dry-run mode:
#   DRY_RUN=1 ./script.sh
DRY_RUN="${DRY_RUN:-0}"

# -----------------------------
# "Goodies"
# -----------------------------
# Nice error message on failure (works well with -E)
trap 'echo "ERROR: line $LINENO: $BASH_COMMAND" >&2' ERR

# Optional: show each command as it runs:
# set -x

# Ensure required binary exists before doing anything
command -v graphregistry >/dev/null 2>&1 || { echo "ERROR: graphregistry not found in PATH" >&2; exit 127; }

run_step() {
  # Usage: run_step "Human label" graphregistry ...
  local label="$1"; shift

  if [[ "$DRY_RUN" == "1" ]]; then
    echo "(dry-run) ulimit -v ${MEM_LIMIT_KIB} && $*" >&2
  else
    # Use && so the command won't run if ulimit fails.
    ( ulimit -v "${MEM_LIMIT_KIB}" && "$@" )
  fi

  printf "%s" "$SEP"
}

#=========================#
# MySQL data update steps #
#=========================#

# run_step "import new data" \
#   graphregistry data import --input_file=scripts/init/sample_sets/epfl_graph_sample_set.json --import_method=object --actions=eval,commit

run_step "airflow reset" \
  graphregistry airflow reset --options=typeflags,airflow,cache

run_step "airflow sync"
  graphregistry airflow sync

run_step "airflow config" \
  graphregistry airflow config --typeflags=@airflow_config.json

run_step "airflow update_checksums" \
  graphregistry airflow update_checksums

run_step "airflow expire" \
  graphregistry airflow expire --older_than=90 --limit_per_type=1000

run_step "airflow refresh" \
  graphregistry airflow refresh --limit_per_type=1000

run_step "airflow status" \
  graphregistry airflow status

run_step "cache update (formulas)" \
  graphregistry cache update --formulas=fields,views,traversals,scores --actions=commit

run_step "cache update (matrix)" \
  graphregistry cache update --matrix --actions=commit,eval,print

run_step "index build" \
  graphregistry index build --actions=commit,eval

run_step "index generate mixed views" \
  graphregistry index mixed_views

run_step "index patch" \
  graphregistry index patch --actions=commit,eval

run_step "airflow rollover" \
  graphregistry airflow rollover --actions=commit

run_step "airflow update_dates" \
  graphregistry airflow update_dates --actions=commit

run_step "airflow reset (again)" \
  graphregistry airflow reset --options=typeflags,airflow

#=================================#
# ElasticSearch data update steps #
#=================================#

# run_step "index generate (elasticsearch)" \
#   graphregistry index generate --target=elasticsearch --index_date=2026-03-13 -r

# run_step "es import" \
#   graphregistry es import --env=xaas_coresrv \
#     --input_folder=/home/dockerhost/data/es_exports/2026-03-13/es_fullindex_2026-03-13 \
#     --rename_to=graphsearch_test_2026-03-13_sample -r

echo "End of script." >&2