Add ability to specify resource_id in the resources.json

configuration/etl/etl.d/ingest_resources.json

@@ -48,7 +48,8 @@
                     "shared_jobs",
                     "timezone",
                     "resource_allocation_type",
-                    "organization"
+                    "organization",
+                    "resource_key"
                 ]
             },
             "destination": {

configuration/etl/etl.d/staging.json

@@ -155,7 +155,8 @@
                         "timezone",
                         "resource_type",
                         "resource_allocation_type",
-                        "organization"
+                        "organization",
+                        "resource_key"
                     ]
                 }
             }
@@ -206,11 +207,6 @@
             "description": "Ingest job PIs",
             "definition_file": "jobs/staging/pi.json"
         },
-        {
-            "name": "resource",
-            "description": "Ingest job resources",
-            "definition_file": "jobs/staging/resource.json"
-        },
         {
             "name": "union-user-pi--pi",
             "description": "Ingest job PIs (combined with users)",

configuration/etl/etl_action_defs.d/common/staging/resource-config.json

@@ -11,7 +11,8 @@
             "timezone": "timezone",
             "type_abbrev": "resource_type",
             "resource_allocation_type_abbrev": "resource_allocation_type",
-            "organization": "organization"
+            "organization": "organization",
+            "resource_key": "resource_key"
         }
     }
 }

configuration/etl/etl_action_defs.d/common/staging/resource.json

@@ -3,19 +3,32 @@
     "#": "cloud resources since a primary key must be assigned to the resource",
     "#": "before the data for that resource is ingested and the cloud data",
     "#": "does not contain the name of the resource.",
+    "#": "If the resource is already in the table the resource_id stays the same. If not then",
+    "#": "the resource_key will used as the resource_id, else the autoincrement",
     "table_definition": {
         "$ref": "${table_definition_dir}/common/staging/resource.json#/table_definition"
     },
     "source_query": {
         "records": {
-            "resource_name": "DISTINCT rc.resource"
+            "resource_id": "COALESCE(res.resource_id, rc.resource_key)",
+            "resource_name": "rc.resource"
         },
         "joins": [
             {
                 "schema": "${SOURCE_SCHEMA}",
                 "name": "staging_resource_config",
                 "alias": "rc"
+            },
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_resource",
+                "type": "LEFT",
+                "on": "rc.resource = res.resource_name",
+                "alias": "res"
             }
+        ],
+        "orderby": [
+            "rc.resource_id"
         ]
     }
 }

configuration/etl/etl_schemas.d/common/resources.schema.json

@@ -45,6 +45,10 @@
             "type": "string",
             "description": "The organization the resource belongs to",
             "maxLength": 100
+        },
+        "resource_key": {
+            "type": "number",
+            "description": "Optional: Numerical identifier for the resource in the database. For a typical XDMoD install this value need not be specified. Typical reasons why you might specify the identifier would be if you had data for the same resource in multiple XDMoD instances and you wanted the internal identifiers to match across all the separate databases."
         }
     },
     "required": [

configuration/etl/etl_tables.d/common/staging/resource-config.json

@@ -57,6 +57,12 @@
                 "type": "varchar(30)",
                 "nullable": true,
                 "comment": "Organization for the resource"
+            },
+            {
+                "name": "resource_key",
+                "type": "int(11)",
+                "nullable": true,
+                "comment": "The numerical identifier for the resource in the XDMoD datawarehouse. For a typical XDMoD install this value need not be specified. Typical reasons why you might specify the identifier would be if you had data for the same resource in multiple XDMoD instances and you wanted the internal identifiers to match across all the separate databases."
             }
         ],
         "indexes": [

docs/configuration.md

@@ -542,6 +542,19 @@ to users.
 ]
 ```
 
+| Property           | Required? | Type    |  Description |
+| ------------------ | --------- | ------- | ------------ |
+| resource                 | Yes | string  | The string identifier for the resource. This is the value that will be used when runing the `xdmod-shredder` command. Typically this should be a short string without spaces |
+| name                     | Yes | string  | The human readable name of the resource. |
+| resource_type            | Yes | string  | The type of the resource from the resource_types.json. Examples include "HPC" for batch computing resources, "Cloud" for infrastructure as a service resources, and "Disk" for data storage resources. |
+| resource_allocation_type | Yes | string  | The base unit of resource allocation, such as by CPU, GPU or Node. There are 4 possible values, CPU, CPUNode, GPU, and GPUNode. CPUNode denotes a resource that allocates nodes of CPUs to users, whereas CPU denotes a resource that allocates individual CPUs to users. |
+| organization             | Yes  | string  | The name of the organization for the resource. It should match the `abbrev` value from the organizations.json file. |
+| description              | No  | string  | Human-readable description of the resource. |
+| pi_column                | No  | string  | The name of the column in the resource specific job table to identify the PI. The column names that may be used with this feature must exist in the corresponding shredded_job_* table (e.g. shredded_job_pbs, shredded_job_slurm) of the mod_shredder database for the resource manager you are using.  For example, to use accounts from PBS/TORQUE you must use "pi_column": "account", but to use accounts from Slurm you must use "pi_column": "account_name". The default is to use the group column |
+| shared_jobs              | No  | boolean | Whether the option indicates that the resource allows multiple to share compute nodes. This information is used by the Job Performance Data (SUPReMM) module to determine which HPC jobs shared compute nodes. The default is that resources are assumed to not allow node sharing. If the SUPReMM module is in use and a resource does allow node sharing then this should be set to true. |
+| timezone                 | No  | string  | The timezone of the resource. This is used in the Job Viewer to display job timestamps in the local timezone of the resource. This value should be a valid timezone identifier such as "America/New_York" and not the three letter abbreviation or hour offset. |
+| resource_key             | No  | integer | Optional specify the internal identifier for the resource in the database. For a typical XDMoD install this value should not be specified. This option exists to support the scenario where there are multiple XDMoD instances (such as production and developement) and you wanted the internal identifiers to match across all the separate databases. |
+
 ### resource_specs.json
 
 Defines resource node and processor counts.  Each object in the array