diff --git a/mmv1/products/vertexai/EvaluationMetric.yaml b/mmv1/products/vertexai/EvaluationMetric.yaml new file mode 100644 index 000000000000..a36b89e1efd9 --- /dev/null +++ b/mmv1/products/vertexai/EvaluationMetric.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: EvaluationMetric +description: | + A reusable metric configuration for Vertex AI evaluation. EvaluationMetrics + define how model outputs are scored, supporting predefined metrics, + LLM-based metrics, pointwise and pairwise comparisons, and custom code + execution metrics. +references: + guides: + 'Official Documentation': 'https://cloud.google.com/vertex-ai/generative-ai/docs/evaluation/overview' + api: 'https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.evaluationMetrics' +docs: +base_url: projects/{{project}}/locations/{{region}}/evaluationMetrics +self_link: projects/{{project}}/locations/{{region}}/evaluationMetrics/{{name}} +create_url: projects/{{project}}/locations/{{region}}/evaluationMetrics?evaluationMetricId={{evaluation_metric_id}} +immutable: true +id_format: projects/{{project}}/locations/{{region}}/evaluationMetrics/{{name}} +import_format: + - projects/{{project}}/locations/{{region}}/evaluationMetrics/{{name}} +timeouts: + insert_minutes: 20 + delete_minutes: 20 +async: + type: OpAsync + operation: + base_url: '{{op_id}}' + actions: ['delete'] +custom_code: +samples: + - name: vertex_ai_evaluation_metric + primary_resource_id: example + steps: + - name: vertex_ai_evaluation_metric + resource_id_vars: + evaluation_metric_id: 'example-metric' + test_env_vars: + project: PROJECT_NAME +parameters: + - name: region + type: String + description: The region of the EvaluationMetric. eg us-central1 + url_param_only: true + required: true + immutable: true + - name: evaluationMetricId + type: String + description: | + The ID to use for the EvaluationMetric, which will become the final + component of the resource name. This value should be 1-63 characters, + and valid characters are /[a-z][0-9]-/. The first character must be + a lowercase letter, and the last character must be a lowercase letter + or number. If not provided, the server will generate a unique ID. + url_param_only: true + immutable: true + default_from_api: true +properties: + - name: name + type: String + description: | + The short name of the EvaluationMetric (the final component of the resource name). + output: true + custom_flatten: templates/terraform/custom_flatten/name_from_self_link.tmpl + - name: displayName + type: String + description: | + The user-friendly display name for the EvaluationMetric. + required: true + - name: description + type: String + description: | + A description of the EvaluationMetric. + - name: metric + type: String + description: | + The metric configuration as a JSON string. Uses camelCase field names + to match the API format. Supports LLM-based metrics and custom code + execution metrics. + See the [API documentation](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/Metric) + for the full schema. + state_func: 'func(v interface{}) string { s, _ := structure.NormalizeJsonString(v); return s }' + custom_flatten: templates/terraform/custom_flatten/json_schema.tmpl + custom_expand: templates/terraform/custom_expand/json_schema.tmpl + validation: + function: validation.StringIsJSON + # The API returns the metric JSON with additional derived fields + # (e.g. metadata.title populated from display_name) that are not part + # of the user's input, causing perpetual diffs. + ignore_read: true + - name: createTime + type: String + description: | + The timestamp of when the EvaluationMetric was created in RFC3339 UTC "Zulu" format, + with nanosecond resolution and up to nine fractional digits. + output: true + - name: updateTime + type: String + description: | + The timestamp of when the EvaluationMetric was last updated in RFC3339 UTC "Zulu" format, + with nanosecond resolution and up to nine fractional digits. + output: true + - name: labels + type: KeyValueLabels + description: | + Labels for the EvaluationMetric. + - name: gcsUri + type: String + description: | + The Google Cloud Storage URI that stores the metric specification. + - name: encryptionSpec + type: NestedObject + description: | + Customer-managed encryption key spec for this EvaluationMetric. If set, + this EvaluationMetric will be secured by this key. + immutable: true + properties: + - name: kmsKeyName + type: String + description: | + Required. The Cloud KMS resource identifier of the customer managed encryption key + used to protect a resource. Has the form: + `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the resource is created. + immutable: true diff --git a/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_evaluation_metric.tf.tmpl b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_evaluation_metric.tf.tmpl new file mode 100644 index 000000000000..7595a04f0b39 --- /dev/null +++ b/mmv1/templates/terraform/samples/services/vertexai/vertex_ai_evaluation_metric.tf.tmpl @@ -0,0 +1,12 @@ +resource "google_vertex_ai_evaluation_metric" "{{$.PrimaryResourceId}}" { + evaluation_metric_id = "{{index $.ResourceIdVars "evaluation_metric_id"}}" + region = "us-central1" + display_name = "Test Evaluation Metric" + description = "An evaluation metric for Terraform acceptance testing" + + metric = jsonencode({ + llmBasedMetricSpec = { + metricPromptTemplate = "Rate the quality of the following response on a scale of 1 to 5. Response: {response}" + } + }) +}