4.5.x distributed improvements

distributed/control-plane/Chart.yaml

@@ -13,5 +13,5 @@ apiVersion: v1
 appVersion: "4.5.0"
 description: A Helm chart for the deployment of WSO2 API Management API Control Plane profile
 name: wso2am-acp
-version: 4.5.0-8
+version: 4.5.0-9
 icon: https://wso2.cachefly.net/wso2/sites/all/images/wso2logo.svg

distributed/control-plane/confs/instance-1/deployment.toml

@@ -113,7 +113,7 @@ password = "$secret{truststore_password}"
 password = {{ .Values.wso2.apim.configurations.security.truststore.password | quote }}
 {{- end }}
 
-{{- range $i, $env := .Values.wso2.apim.configurations.gateway.environments }}
+{{ range $i, $env := .Values.wso2.apim.configurations.gateway.environments }}
 [[apim.gateway.environment]]
 name = {{ $env.name | quote }}
 type = {{ $env.type | quote }}
@@ -135,7 +135,17 @@ https_endpoint = "https://{{ $env.httpHostname }}"
 websub_event_receiver_http_endpoint = "http://{{ $env.websubHostname }}"
 websub_event_receiver_https_endpoint = "https://{{ $env.websubHostname }}"
 
-{{- end }}
+{{ range $j, $vhostenv := $env.virtualHosts }}
+[[apim.gateway.environment.virtual_host]]
+ws_endpoint = "ws://{{ $vhostenv.wsHostname }}"
+wss_endpoint = "wss://{{ $vhostenv.wsHostname }}"
+http_endpoint = "http://{{ $vhostenv.httpHostname }}"
+https_endpoint = "https://{{ $vhostenv.httpHostname }}"
+websub_event_receiver_http_endpoint = "http://{{ $vhostenv.websubHostname }}"
+websub_event_receiver_https_endpoint = "https://{{ $vhostenv.websubHostname }}"
+{{ end }}
+
+{{ end }}
 
 [apim.key_manager]
 {{- if .Values.wso2.apim.configurations.iskm.enabled }}
@@ -249,11 +259,12 @@ auth_urls = ["ssl://{{ template "apim-helm-cp.fullname" . }}-2-service:{{ add 97
 {{- if .Values.wso2.apim.configurations.iskm.enabled }}
 revoke_endpoint = "https://{{ .Values.wso2.apim.configurations.iskm.serviceName }}:{{ .Values.wso2.apim.configurations.iskm.servicePort }}/oauth2/revoke"
 {{- else }}
-revoke_endpoint = "https://{{ template "apim-helm-cp.fullname" . }}-service:{{ add 9443 .Values.wso2.apim.portOffset }}/oauth2/revoke"
+revoke_endpoint = "https://{{ .Values.wso2.apim.configurations.km.serviceName }}:{{ .Values.wso2.apim.configurations.km.servicePort }}/oauth2/revoke"
 {{- end }}
 enable_token_encryption = {{ .Values.wso2.apim.configurations.oauth_config.enableTokenEncryption }}
 enable_token_hashing = {{ .Values.wso2.apim.configurations.oauth_config.enableTokenHashing }}
 allowed_scopes = {{ toJson .Values.wso2.apim.configurations.oauth_config.allowedScopes }}
+#enable_outbound_auth_header = true
 
 {{- if .Values.wso2.apim.configurations.openTracer.enabled }}
 [apim.open_tracer]
@@ -271,6 +282,12 @@ remote_tracer.hostname = {{ .Values.wso2.apim.configurations.openTelemetry.hostn
 remote_tracer.port = {{ .Values.wso2.apim.configurations.openTelemetry.port | quote }}
 {{- end }}
 
+{{- if .Values.wso2.apim.configurations.jmx.enabled }}
+[monitoring.jmx]
+rmi_registry_port = {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+rmi_server_port = {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+{{- end }}
+
 [apim.ai]
 {{- if .Values.wso2.apim.configurations.ai.enabled }}
 enable = true

distributed/control-plane/confs/instance-2/deployment.toml

@@ -113,7 +113,7 @@ password = "$secret{truststore_password}"
 password = {{ .Values.wso2.apim.configurations.security.truststore.password | quote }}
 {{- end }}
 
-{{- range $i, $env := .Values.wso2.apim.configurations.gateway.environments }}
+{{ range $i, $env := .Values.wso2.apim.configurations.gateway.environments }}
 [[apim.gateway.environment]]
 name = {{ $env.name | quote }}
 type = {{ $env.type | quote }}
@@ -135,7 +135,17 @@ https_endpoint = "https://{{ $env.httpHostname }}"
 websub_event_receiver_http_endpoint = "http://{{ $env.websubHostname }}"
 websub_event_receiver_https_endpoint = "https://{{ $env.websubHostname }}"
 
-{{- end }}
+{{ range $j, $vhostenv := $env.virtualHosts }}
+[[apim.gateway.environment.virtual_host]]
+ws_endpoint = "ws://{{ $vhostenv.wsHostname }}"
+wss_endpoint = "wss://{{ $vhostenv.wsHostname }}"
+http_endpoint = "http://{{ $vhostenv.httpHostname }}"
+https_endpoint = "https://{{ $vhostenv.httpHostname }}"
+websub_event_receiver_http_endpoint = "http://{{ $vhostenv.websubHostname }}"
+websub_event_receiver_https_endpoint = "https://{{ $vhostenv.websubHostname }}"
+{{ end }}
+
+{{ end }}
 
 [apim.key_manager]
 {{- if .Values.wso2.apim.configurations.iskm.enabled }}
@@ -249,11 +259,12 @@ auth_urls = ["ssl://{{ template "apim-helm-cp.fullname" . }}-2-service:{{ add 97
 {{- if .Values.wso2.apim.configurations.iskm.enabled }}
 revoke_endpoint = "https://{{ .Values.wso2.apim.configurations.iskm.serviceName }}:{{ .Values.wso2.apim.configurations.iskm.servicePort }}/oauth2/revoke"
 {{- else }}
-revoke_endpoint = "https://{{ template "apim-helm-cp.fullname" . }}-service:{{ add 9443 .Values.wso2.apim.portOffset }}/oauth2/revoke"
+revoke_endpoint = "https://{{ .Values.wso2.apim.configurations.km.serviceName }}:{{ .Values.wso2.apim.configurations.km.servicePort }}/oauth2/revoke"
 {{- end }}
 enable_token_encryption = {{ .Values.wso2.apim.configurations.oauth_config.enableTokenEncryption }}
 enable_token_hashing = {{ .Values.wso2.apim.configurations.oauth_config.enableTokenHashing }}
 allowed_scopes = {{ toJson .Values.wso2.apim.configurations.oauth_config.allowedScopes }}
+#enable_outbound_auth_header = true
 
 {{- if .Values.wso2.apim.configurations.openTracer.enabled }}
 [apim.open_tracer]
@@ -271,6 +282,12 @@ remote_tracer.hostname = {{ .Values.wso2.apim.configurations.openTelemetry.hostn
 remote_tracer.port = {{ .Values.wso2.apim.configurations.openTelemetry.port | quote }}
 {{- end }}
 
+{{- if .Values.wso2.apim.configurations.jmx.enabled }}
+[monitoring.jmx]
+rmi_registry_port = {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+rmi_server_port = {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+{{- end }}
+
 [apim.ai]
 {{- if .Values.wso2.apim.configurations.ai.enabled }}
 enable = true

distributed/control-plane/confs/log4j2.properties

@@ -6,6 +6,10 @@ appenders = {{ .Values.wso2.apim.log4j2.appenders }}, CARBON_CONSOLE,HTTP_ACCESS
 appenders = CARBON_CONSOLE,HTTP_ACCESS_CONSOLE,AUDIT_CONSOLE,TRANSACTION_CONSOLE,CORRELATION_CONSOLE,CARBON_LOGFILE,AUDIT_LOGFILE,ATOMIKOS_LOGFILE,CARBON_TRACE_LOGFILE,ERROR_LOGFILE,OPEN_TRACING,SERVICE_APPENDER,TRACE_APPENDER,osgi,CORRELATION,BOTDATA_APPENDER,API_LOGFILE, APIM_GATEWAY_ACCESS_APPENDER, APIM_GOV_AUDIT_LOGFILE
 {{- end }}
 
+{{- if .Values.wso2.apim.log4j2.appendersConfig }}
+{{ .Values.wso2.apim.log4j2.appendersConfig }}
+{{- end }}
+
 # CARBON_CONSOLE is set to be a ConsoleAppender using a PatternLayout.
 appender.CARBON_CONSOLE.type = Console
 appender.CARBON_CONSOLE.name = CARBON_CONSOLE
@@ -269,6 +273,10 @@ loggers = {{ .Values.wso2.apim.log4j2.loggers }}, AUDIT_LOG, trace-messages, org
 loggers = AUDIT_LOG, trace-messages, org-apache-coyote, com-hazelcast, Owasp-CsrfGuard, org-apache-axis2-wsdl-codegen-writer-PrettyPrinter, org-apache-axis2-clustering, org-apache-catalina, org-apache-tomcat, org-wso2-carbon-apacheds, org-apache-directory-server-ldap, org-apache-directory-server-core-event, com-atomikos, org-quartz, org-apache-jackrabbit-webdav, org-apache-juddi, org-apache-commons-digester-Digester, org-apache-jasper-compiler-TldLocationsCache, org-apache-qpid, org-apache-qpid-server-Main, qpid-message, qpid-message-broker-listening, org-apache-tiles, org-apache-commons-httpclient, org-apache-solr, me-prettyprint-cassandra-hector-TimingLogger, org-apache-axis-enterprise, org-apache-directory-shared-ldap, org-apache-directory-server-ldap-handlers, org-apache-directory-shared-ldap-entry-DefaultServerAttribute, org-apache-directory-server-core-DefaultDirectoryService, org-apache-directory-shared-ldap-ldif-LdifReader, org-apache-directory-server-ldap-LdapProtocolHandler, org-apache-directory-server-core, org-apache-directory-server-ldap-LdapSession, DataNucleus, Datastore, Datastore-Schema, JPOX-Datastore, JPOX-Plugin, JPOX-MetaData, JPOX-Query, JPOX-General, JPOX-Enhancer, org-apache-hadoop-hive, hive, ExecMapper, ExecReducer, net-sf-ehcache-config-ConfigurationFactory, axis2Deployment, equinox, tomcat2, StAXDialectDetector, trace, synapse, synapse_transport, axis2, axis2_transport, org-wso2-carbon, hunsicker, thrift-publisher, service_logger, trace_logger, org-wso2-carbon-apimgt-gateway-mediators-BotDetectionMediator, correlation, GatewayAccessLogger
 {{- end }}
 
+{{- if .Values.wso2.apim.log4j2.loggersConfig }}
+{{ .Values.wso2.apim.log4j2.loggersConfig }}
+{{- end }}
+
 logger.API_LOG.name = API_LOG
 logger.API_LOG.level = INFO
 logger.API_LOG.appenderRef.API_LOGFILE.ref = API_LOGFILE
@@ -565,3 +573,10 @@ logger.org-wso2-carbon-apimgt-gateway-mediators-BotDetectionMediator.appenderRef
 logger.org-wso2-carbon-apimgt-gateway-mediators-BotDetectionMediator.additivity = false
 
 category.SERVICE_APPENDER._OpenService_ = TRACE_APPENDER, BOTDATA_APPENDER
+
+logger.auth-framework.name = org.wso2.carbon.identity.application.authentication.framework
+logger.auth-framework.level = DEBUG
+logger.auth-framework.appenderRef.CARBON_LOGFILE.ref = CARBON_LOGFILE
+logger.oidc-authenticator.name = org.wso2.carbon.identity.application.authenticator.oidc
+logger.oidc-authenticator.level = DEBUG
+logger.oidc-authenticator.appenderRef.CARBON_LOGFILE.ref = CARBON_LOGFILE

distributed/control-plane/templates/control-plane/instance-1/wso2am-cp-deployment.yaml

@@ -59,21 +59,32 @@ spec:
       {{- else if .Values.azure.enabled }}
       serviceAccount: {{ .Values.azure.serviceAccountName }}
       {{- end }}
+      {{- else if .Values.wso2.deployment.serviceAccountName }}
+      serviceAccount: {{ .Values.wso2.deployment.serviceAccountName }}
       {{- end }}
       securityContext:
         seccompProfile:
           type: {{ .Values.kubernetes.securityContext.seccompProfile.type }}
           {{- if eq .Values.kubernetes.securityContext.seccompProfile.type "Localhost" }}
           localhostProfile: {{ .Values.kubernetes.securityContext.seccompProfile.localhostProfile }}
           {{- end }}
-      {{- if and .Values.wso2.deployment.image.imagePullSecrets.enabled }}
       imagePullSecrets:
-      - name: {{ template "apim-helm-cp.fullname" . }}-docker-registry-auth
-      {{- end }}
+        {{- if .Values.wso2.deployment.image.imagePullSecrets.enabled }}
+        - name: {{ template "apim-helm-cp.fullname" . }}-docker-registry-auth
+        {{- end }}
+        {{- if .Values.wso2.deployment.image.imagePullSecrets.secrets }}
+        {{- range .Values.wso2.deployment.image.imagePullSecrets.secrets }}
+        - name: {{ . }}
+        {{- end }}
+        {{- end }}
       containers:
       - name: wso2am-control-plane
+        {{ if .Values.wso2.deployment.image.tag }}
+        image: {{ .Values.wso2.deployment.image.registry }}/{{ .Values.wso2.deployment.image.repository }}:{{ .Values.wso2.deployment.image.tag }}
+        {{- else }}
         image: {{ .Values.wso2.deployment.image.registry }}/{{ .Values.wso2.deployment.image.repository }}@{{ .Values.wso2.deployment.image.digest }}
-        imagePullPolicy: {{ .Values.wso2.deployment.imagePullPolicy }}
+        {{- end }}
+        imagePullPolicy: {{ .Values.wso2.deployment.image.imagePullPolicy }}
         env:
         - name: PROFILE_NAME
           value: default
@@ -84,6 +95,12 @@ spec:
               fieldPath: status.podIP
         - name: JVM_MEM_OPTS
           value: "-Xms{{ .Values.wso2.deployment.resources.jvm.memory.xms }} -Xmx{{ .Values.wso2.deployment.resources.jvm.memory.xmx }}"
+        {{- if .Values.wso2.deployment.envs }}
+        {{- range $name, $value := .Values.wso2.deployment.envs }}
+        - name: {{ $name }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
         {{- if and .Values.wso2.apim.configurations.existingSecret.secretName .Values.wso2.apim.configurations.existingSecret.adminPasswordKey }}
         - name: WSO2_ADMIN_PASSWORD
           valueFrom:
@@ -167,6 +184,12 @@ spec:
             protocol: "TCP"
           - containerPort: {{ add 5672 .Values.wso2.apim.portOffset }}
             protocol: "TCP"
+          {{- if .Values.wso2.apim.configurations.jmx.enabled }}
+          - containerPort: {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+            protocol: "TCP"
+          - containerPort: {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+            protocol: "TCP"
+          {{- end }}
         volumeMounts:
         - name: wso2am-control-plane-conf
           mountPath: /home/wso2carbon/wso2-config-volume/repository/conf/deployment.toml

distributed/control-plane/templates/control-plane/instance-1/wso2am-cp-service.yaml

@@ -33,3 +33,11 @@ spec:
     - name: servlet-https
       protocol: TCP
       port: {{ add 9443 .Values.wso2.apim.portOffset }}
+    {{- if .Values.wso2.apim.configurations.jmx.enabled }}
+    - name: jmx-registry-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+    - name: jmx-server-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+    {{- end }}

distributed/control-plane/templates/control-plane/instance-2/wso2am-cp-deployment.yaml

@@ -60,21 +60,32 @@ spec:
       {{- else if .Values.azure.enabled }}
       serviceAccount: {{ .Values.azure.serviceAccountName }}
       {{- end }}
+      {{- else if .Values.wso2.deployment.serviceAccountName }}
+      serviceAccount: {{ .Values.wso2.deployment.serviceAccountName }}
       {{- end }}
       securityContext:
         seccompProfile:
           type: {{ .Values.kubernetes.securityContext.seccompProfile.type }}
           {{- if eq .Values.kubernetes.securityContext.seccompProfile.type "Localhost" }}
           localhostProfile: {{ .Values.kubernetes.securityContext.seccompProfile.localhostProfile }}
           {{- end }}
-      {{- if and .Values.wso2.deployment.image.imagePullSecrets.enabled }}
       imagePullSecrets:
-      - name: {{ template "apim-helm-cp.fullname" . }}-docker-registry-auth
-      {{- end }}
+        {{- if .Values.wso2.deployment.image.imagePullSecrets.enabled }}
+        - name: {{ template "apim-helm-cp.fullname" . }}-docker-registry-auth
+        {{- end }}
+        {{- if .Values.wso2.deployment.image.imagePullSecrets.secrets }}
+        {{- range .Values.wso2.deployment.image.imagePullSecrets.secrets }}
+        - name: {{ . }}
+        {{- end }}
+        {{- end }}
       containers:
       - name: wso2am-control-plane
+        {{ if .Values.wso2.deployment.image.tag }}
+        image: {{ .Values.wso2.deployment.image.registry }}/{{ .Values.wso2.deployment.image.repository }}:{{ .Values.wso2.deployment.image.tag }}
+        {{- else }}
         image: {{ .Values.wso2.deployment.image.registry }}/{{ .Values.wso2.deployment.image.repository }}@{{ .Values.wso2.deployment.image.digest }}
-        imagePullPolicy: {{ .Values.wso2.deployment.imagePullPolicy }}
+        {{- end }}
+        imagePullPolicy: {{ .Values.wso2.deployment.image.imagePullPolicy }}
         env:
         - name: PROFILE_NAME
           value: default
@@ -85,6 +96,12 @@ spec:
               fieldPath: status.podIP
         - name: JVM_MEM_OPTS
           value: "-Xms{{ .Values.wso2.deployment.resources.jvm.memory.xms }} -Xmx{{ .Values.wso2.deployment.resources.jvm.memory.xmx }}"
+        {{- if .Values.wso2.deployment.envs }}
+        {{- range $name, $value := .Values.wso2.deployment.envs }}
+        - name: {{ $name }}
+          value: {{ $value | quote }}
+        {{- end }}
+        {{- end }}
         {{- if and .Values.wso2.apim.configurations.existingSecret.secretName .Values.wso2.apim.configurations.existingSecret.adminPasswordKey }}
         - name: WSO2_ADMIN_PASSWORD
           valueFrom:
@@ -168,6 +185,12 @@ spec:
             protocol: "TCP"
           - containerPort: {{ add 5672 .Values.wso2.apim.portOffset }}
             protocol: "TCP"
+          {{- if .Values.wso2.apim.configurations.jmx.enabled }}
+          - containerPort: {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+            protocol: "TCP"
+          - containerPort: {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+            protocol: "TCP"
+          {{- end }}
         volumeMounts:
         - name: wso2am-control-plane-conf
           mountPath: /home/wso2carbon/wso2-config-volume/repository/conf/deployment.toml

distributed/control-plane/templates/control-plane/instance-2/wso2am-cp-service.yaml

@@ -34,4 +34,12 @@ spec:
     - name: servlet-https
       protocol: TCP
       port: {{ add 9443 .Values.wso2.apim.portOffset }}
+    {{- if .Values.wso2.apim.configurations.jmx.enabled }}
+    - name: jmx-registry-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+    - name: jmx-server-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+    {{- end }}
 {{- end }}

distributed/control-plane/templates/control-plane/wso2am-cp-service.yaml

@@ -26,3 +26,11 @@ spec:
     - name: servlet-https
       protocol: TCP
       port: {{ add 9443 .Values.wso2.apim.portOffset }}
+    {{- if .Values.wso2.apim.configurations.jmx.enabled }}
+    - name: jmx-registry-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiRegistryPort }}
+    - name: jmx-server-port
+      protocol: TCP
+      port: {{ .Values.wso2.apim.configurations.jmx.rmiServerPort }}
+    {{- end }}