Skip to content

Storage design #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jagadeeshnv wants to merge 10 commits into
base: staging_slurm_no_nfs
Choose a base branch
from
Open

Storage design #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
444cd7b
Propdal for new generic storage
jagadeeshnv Mar 12, 2026
d8e0103
Added L1 validation for the inputs
jagadeeshnv Mar 12, 2026
b2ed62f
Ipdated l1 validation
jagadeeshnv Mar 12, 2026
d03fe80
Updated design to remove implementation
jagadeeshnv Mar 12, 2026
038423c
Updated design
jagadeeshnv Mar 20, 2026
cc352a6
storage_config updates with proper vast powervault examples
jagadeeshnv Mar 20, 2026
04dd2d0
Enhanced examples for storage mounts
jagadeeshnv Mar 20, 2026
d233ab5
STorage design almost finished
jagadeeshnv Mar 27, 2026
46f6037
storage design which looks good
jagadeeshnv Mar 28, 2026
4897f7b
Storage design final
jagadeeshnv Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,328 changes: 1,328 additions & 0 deletions STORAGE_DESIGN.md
View file
Open in desktop

Large diffs are not rendered by default.

325 changes: 265 additions & 60 deletions common/library/module_utils/input_validation/schema/storage_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,88 +3,293 @@
"title": "Configuration Schema",
"type": "object",
"properties": {
"nfs_client_params": {
"powervault_config": {
"type": "array",
"description": "List of PowerVault iSCSI volume connection definitions. Processed via runcmd script because device path is only known after iSCSI login + multipath scan.",
"items": {
"type": "object",
"properties": {
"nfs_name": {
"name": {
"type": "string",
"description": "Unique identifier for this PowerVault volume",
"pattern": "^[a-zA-Z0-9_-]+$",
"minLength": 1,
"maxLength": 64
},
"ip": {
"description": "List of target controller IP addresses for iSCSI discovery",
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"format": "ipv4"
},
"uniqueItems": true
},
"port": {
"description": "TCP port for iSCSI target (default 3260)",
"type": "integer",
"minimum": 1,
"maximum": 65535
},
"iscsi_initiator": {
"description": "iSCSI initiator IQN",
"type": "string",
"description": "The unique NFS server name"
"pattern": "^iqn\\.[a-zA-Z0-9.-]+(?::[a-zA-Z0-9._:-]+)?$"
},
"server_ip": {
"volume_id": {
"description": "Volume identifier (hex string / WWN) for multipath device matching",
"type": "string",
"anyOf": [
{
"allOf": [
{ "pattern": ".*[A-Za-z].*" },
{ "format": "idn-hostname" }
]
},
{
"allOf": [
{ "pattern": "^[0-9.]+$" },
{ "format": "ipv4" }
]
}
]
"pattern": "^[a-fA-F0-9]+$"
},
"server_share_path": {
"mount_point": {
"type": "string",
"pattern": "^/(?:[^/]+(?:/[^/]+)*)?/?$"
"description": "Where the discovered device gets mounted",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"client_share_path": {
"mount_params": {
"type": "string",
"pattern": "^/(?:[^/]+(?:/[^/]+)*)?/?$"
"description": "Named profile for fs_type/mnt_opts (read by the runcmd script)",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"client_mount_options": {
"type": "string"
"node_key": {
"type": "string",
"description": "ds.meta_data key for per-node bind mounts (e.g., local_hostname). When present, generates bind mounts under mount_point/<node_key_value>/",
"enum": ["local_hostname", "local_ipv4", "instance_id"]
},
"node_mount_point": {
"type": "array",
"description": "List of bind mount target paths. Required when node_key is set. Each gets: mount_point/<node_key_value>/<target_stripped_slash> -> <target>",
"items": {
"type": "string",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"minItems": 1,
"uniqueItems": true
},

"functional_group_prefix": {
"type": "array",
"description": "List of functional group prefixes for node targeting",
"items": {
"type": "string",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"uniqueItems": true
}
},
"required": [
"server_ip",
"server_share_path",
"client_share_path",
"client_mount_options"
]
},
"minItems": 1
"required": ["name", "ip", "iscsi_initiator", "volume_id", "mount_point", "functional_group_prefix"],
"additionalProperties": false
}
},
"powervault_config": {
"type": "object",
"required": ["ip", "iscsi_initiator", "volume_id"],
"properties": {
"ip": {
"description": "List of target controller IP addresses",
"type": "array",
"minItems": 1,
"items": {
"beegfs_config": {
"type": "array",
"description": "List of BeeGFS parallel filesystem mount definitions. BeeGFS uses beegfs-mounts.conf and beegfs-client service, not fstab. Each entry represents one BeeGFS cluster connection.",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Unique identifier for this BeeGFS mount",
"pattern": "^[a-zA-Z0-9_-]+$",
"minLength": 1,
"maxLength": 64
},
"mgmtd_host": {
"type": "string",
"description": "IP address of the BeeGFS management server",
"format": "ipv4"
},
"uniqueItems": true
},

"port": {
"description": "TCP port for iSCSI (default 3260)",
"type": "integer"
"mount_point": {
"type": "string",
"description": "Client mount location for this BeeGFS filesystem",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"conn_auth_file": {
"type": "string",
"description": "Path to the BeeGFS connauth shared secret file on the omnia_core. Copied to /etc/beegfs/ on target nodes.",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"client_conf_overrides": {
"type": "object",
"description": "Optional key-value overrides for /etc/beegfs/beegfs-client.conf (e.g., tuneNumWorkers, connMaxInternodeNum)",
"additionalProperties": {
"type": ["string", "integer", "boolean"]
}
},
"functional_group_prefix": {
"type": "array",
"description": "List of functional group prefixes for node targeting",
"items": {
"type": "string",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"uniqueItems": true
}
},

"iscsi_initiator": {
"description": "iSCSI initiator IQN",
"type": "string",
"pattern": "^iqn\\.[a-zA-Z0-9.-]+(?::[a-zA-Z0-9._:-]+)?$"
"required": ["name", "mgmtd_host", "mount_point", "conn_auth_file", "functional_group_prefix"],
"additionalProperties": false
}
},
"mounts": {
"type": "array",
"description": "Cloud-init compatible mount configurations. Source must be known at boot time.",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Unique identifier for this mount entry",
"pattern": "^[a-zA-Z0-9_-]+$",
"minLength": 1,
"maxLength": 64
},
"source": {
"type": "string",
"description": "Device name or network path (e.g., /dev/sdc, UUID=xxx, 192.168.1.100:/export/share, powervault:<name>)",
"minLength": 1
},
"mount_point": {
"type": "string",
"description": "Mount point path",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"fs_type": {
"type": "string",
"description": "Filesystem type. Overrides mount_params profile when specified.",
"enum": ["auto", "ext2", "ext3", "ext4", "xfs", "btrfs", "nfs", "nfs4", "cifs", "tmpfs", "cephfs", "vfat", "ntfs", "none", "beegfs", "fuse.s3fs"]
},
"mnt_opts": {
"type": "string",
"description": "Mount options. Overrides mount_params profile when specified.",
"pattern": "^[a-zA-Z0-9,=._@/-]+$"
},
"dump_freq": {
"type": "string",
"description": "Dump frequency (usually 0). Overrides mount_params profile when specified.",
"pattern": "^[0-2]$"
},
"fsck_pass": {
"type": "string",
"description": "Fsck pass number (usually 0 or 2). Overrides mount_params profile when specified.",
"pattern": "^[0-9]$"
},
"mount_params": {
"type": "string",
"description": "Name of the mount_params profile to use for unspecified fields",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"node_key": {
"type": "string",
"description": "ds.meta_data key for per-node bind mounts. When present, fs_type is forced to none and mnt_opts to bind.",
"enum": ["local_hostname", "local_ipv4", "instance_id"]
},
"node_mount_point": {
"type": "array",
"description": "List of bind mount target paths. Required when node_key is set.",
"items": {
"type": "string",
"pattern": "^/[a-zA-Z0-9/_.-]*$"
},
"minItems": 1,
"uniqueItems": true
},
"functional_group_prefix": {
"type": "array",
"description": "List of oChaMI functional group prefixes to apply this mount to. Omit to apply to all groups.",
"items": {
"type": "string",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"uniqueItems": true
},
"role": {
"type": "string",
"description": "Omnia-reserved role for this mount. Used for internal infrastructure mounts.",
"enum": ["omnia_slurm_share", "omnia_k8s_share"]
}
},

"volume_id": {
"description": "Volume identifier (hex string)",
"type": "string",
"pattern": "^[a-fA-F0-9]+$"
"required": ["name", "source", "mount_point"],
"additionalProperties": false
}
},
"mount_params": {
"type": "object",
"description": "Named mount parameter profiles. Each profile provides defaults for fs_type, mnt_opts, dump_freq, fsck_pass. Custom fields are allowed for backend-specific metadata.",
"patternProperties": {
"^[a-zA-Z0-9_-]+$": {
"type": "object",
"properties": {
"fs_type": {
"type": "string",
"description": "Default filesystem type",
"enum": ["auto", "ext2", "ext3", "ext4", "xfs", "btrfs", "nfs", "nfs4", "cifs", "tmpfs", "cephfs", "vfat", "ntfs", "none", "beegfs", "fuse.s3fs"]
},
"mnt_opts": {
"type": "string",
"description": "Default mount options",
"pattern": "^[a-zA-Z0-9,=._@/-]+$"
},
"dump_freq": {
"type": "string",
"description": "Default dump frequency",
"pattern": "^[0-2]$"
},
"fsck_pass": {
"type": "string",
"description": "Default fsck pass number",
"pattern": "^[0-9]$"
}
},
"required": ["fs_type", "mnt_opts"],
"additionalProperties": true
}
},
"additionalProperties": false
},
"swap": {
"type": "array",
"description": "Swap file configurations",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Unique identifier for this swap entry",
"pattern": "^[a-zA-Z0-9_-]+$",
"minLength": 1,
"maxLength": 64
},
"filename": {
"type": "string",
"description": "Path to the swap file to create",
"pattern": "^/[a-zA-Z0-9/_.-]+$"
},
"size": {
"type": "string",
"description": "Size in bytes, 'auto', or human-readable format (e.g., 2G, 512M)",
"pattern": "^(auto|[0-9]+[BKMGT]?)$"
},
"maxsize": {
"type": "string",
"description": "Maximum size (used with size: auto)",
"pattern": "^[0-9]+[BKMGT]?$"
},
"functional_group_prefix": {
"type": "array",
"description": "List of oChaMI functional group prefixes to apply this swap to. Omit to apply to all groups.",
"items": {
"type": "string",
"pattern": "^[a-zA-Z0-9_-]+$"
},
"uniqueItems": true
}
},
"required": ["name", "filename", "size"],
"additionalProperties": false
}
}
},
"required": [
"nfs_client_params"
]
"required": [],
"additionalProperties": false
}
8 changes: 7 additions & 1 deletion discovery/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,12 @@

echo "[INFO] ===== Completed slurmd setup (aarch64) ====="

- path: /etc/munge/munge.key
owner: {{ munge_user }}:{{ munge_group }}
permissions: '{{ file_mode_400 }}'
encoding: b64
content: {{ slurp_munge_key.content }}

- path: /usr/local/bin/configure_munge_and_pam.sh
permissions: '{{ file_mode_755 }}'
content: |
Expand Down Expand Up @@ -442,7 +448,7 @@
content: |
{% for key in ip_name_map | sort %}
{{ ip_name_map[key] }} {{ key }}
{% endfor %}
{%- endfor %}

- path: /etc/sysconfig/slurmd
owner: root:root
Expand Down
Loading
Loading