Service Telemetry Framework 1.5

crc delete

crc start --memory=49152 --cpus=8

$ oc get sub --all-namespaces -o json | jq '.items[] | select(.metadata.name | match("cert-manager")) | .metadata.name'

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  name: default
  namespace: service-telemetry
spec:
  [...]
  backends:
    metrics:
      prometheus:
        enabled: true

$ oc get storageclasses
NAME                 PROVISIONER                RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
csi-manila-ceph      manila.csi.openstack.org   Delete          Immediate              false                  20h
standard (default)   kubernetes.io/cinder       Delete          WaitForFirstConsumer   true                   20h
standard-csi         cinder.csi.openstack.org   Delete          WaitForFirstConsumer   true                   20h

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  name: default
  namespace: service-telemetry
spec:
  [...]
  backends:
    metrics:
      prometheus:
        enabled: true
        storage:
          strategy: persistent
          persistent:
            storageClass: standard-csi
            pvcStorageRequest: 50G

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  name: default
  namespace: service-telemetry
spec:
  [...]
  backends:
    events:
      elasticsearch:
        enabled: true
        forwarding:
          hostUrl: https://external-elastic-http.domain:9200
          tlsServerName: ""
          tlsSecretName: elasticsearch-es-cert
          userSecretName: elasticsearch-es-elastic-user
          useBasicAuth: true
          useTls: true

$ oc create secret generic elasticsearch-es-elastic-user --from-literal=elastic='<PASSWORD>'

$ cat EXTERNAL-ES-CA.pem
-----BEGIN CERTIFICATE-----
[...]
-----END CERTIFICATE-----

$ oc create secret generic elasticsearch-es-cert --from-file=ca.crt=EXTERNAL-ES-CA.pem

$ oc get secrets

$ oc get secret/default-interconnect-selfsigned -o jsonpath='{.data.ca\.crt}' | base64 -d

$ oc project service-telemetry

$ oc get secret default-interconnect-users -o json | jq -r .data.guest | base64 -d

$ oc project service-telemetry

$ oc get routes -ogo-template='{{ range .items }}{{printf "%s\n" .spec.host }}{{ end }}' | grep "\-5671"
default-interconnect-5671-service-telemetry.apps.infra.watch

resource_registry:
  OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml

parameter_defaults:
    ExtraConfig:
        qdr::router_id: "%{::hostname}.cloud1"

    MetricsQdrConnectors:
        - host: default-interconnect-5671-service-telemetry.apps.infra.watch
          port: 443
          role: edge
          verifyHostname: false
          sslProfile: sslProfile
          saslUsername: guest@default-interconnect
          saslPassword: pass:<password_from_stf>

    MetricsQdrSSLProfiles:
        - name: sslProfile
          caCertFileContent: |
            -----BEGIN CERTIFICATE-----
            <snip>
            -----END CERTIFICATE-----

    CeilometerQdrMetricsConfig:
        driver: amqp
        topic: cloud1-metering

    CollectdAmqpInstances:
        cloud1-telemetry:
            format: JSON
            presettle: false

    CollectdSensubilityResultsChannel: sensubility/cloud1-telemetry

$ source ~/stackrc

(undercloud)$ openstack overcloud deploy --templates \
 -e [your environment files] \
 -e /usr/share/openstack-tripleo-heat-templates/environments/metrics/ceilometer-write-qdr.yaml \
 -e /usr/share/openstack-tripleo-heat-templates/environments/metrics/qdr-edge-only.yaml \
 -e /home/stack/enable-stf.yaml \
 -e /home/stack/stf-connectors.yaml

$ sudo podman container inspect --format '{{.State.Status}}' metrics_qdr collectd ceilometer_agent_notification ceilometer_agent_central
running
running
running
running

$ sudo podman exec -it metrics_qdr cat /etc/qpid-dispatch/qdrouterd.conf

listener {
    host: 172.17.1.44
    port: 5666
    authenticatePeer: no
    saslMechanisms: ANONYMOUS
}

$ sudo podman exec -it metrics_qdr qdstat --bus=172.17.1.44:5666 --connections

Connections
  id   host                                                                  container                                                                                                  role    dir  security                            authentication  tenant
  ============================================================================================================================================================================================================================================================================================
  1    default-interconnect-5671-service-telemetry.apps.infra.watch:443      default-interconnect-7458fd4d69-bgzfb                                                                      edge    out  TLSv1.2(DHE-RSA-AES256-GCM-SHA384)  anonymous-user
  12   172.17.1.44:60290                                                     openstack.org/om/container/controller-0/ceilometer-agent-notification/25/5c02cee550f143ec9ea030db5cccba14  normal  in   no-security                         no-auth
  16   172.17.1.44:36408                                                     metrics                                                                                                    normal  in   no-security                         anonymous-user
  899  172.17.1.44:39500                                                     10a2e99d-1b8a-4329-b48c-4335e5f75c84                                                                       normal  in   no-security                         no-auth

$ sudo podman exec -it metrics_qdr qdstat --bus=172.17.1.44:5666 --links
Router Links
  type      dir  conn id  id    peer  class   addr                  phs  cap  pri  undel  unsett  deliv    presett  psdrop  acc  rej  rel     mod  delay  rate
  ===========================================================================================================================================================
  endpoint  out  1        5           local   _edge                      250  0    0      0       2979926  0        0       0    0    2979926 0    0      0
  endpoint  in   1        6                                              250  0    0      0       0        0        0       0    0    0       0    0      0
  endpoint  in   1        7                                              250  0    0      0       0        0        0       0    0    0       0    0      0
  endpoint  out  1        8                                              250  0    0      0       0        0        0       0    0    0       0    0      0
  endpoint  in   1        9                                              250  0    0      0       0        0        0       0    0    0       0    0      0
  endpoint  out  1        10                                             250  0    0      0       911      911      0       0    0    0       0    911    0
  endpoint  in   1        11                                             250  0    0      0       0        911      0       0    0    0       0    0      0
  endpoint  out  12       32          local   temp.lSY6Mcicol4J2Kp       250  0    0      0       0        0        0       0    0    0       0    0      0
  endpoint  in   16       41                                             250  0    0      0       2979924  0        0       0    0    2979924 0    0      0
  endpoint  in   912      1834        mobile  $management           0    250  0    0      0       1        0        0       1    0    0       0    0      0
  endpoint  out  912      1835        local   temp.9Ok2resI9tmt+CT       250  0    0      0       0        0        0       0    0    0       0    0      0

$ oc get pods -l application=default-interconnect

NAME                                    READY   STATUS    RESTARTS   AGE
default-interconnect-7458fd4d69-bgzfb   1/1     Running   0          6d21h

$ oc exec -it deploy/default-interconnect -- qdstat --connections

2020-04-21 18:25:47.243852 UTC
default-interconnect-7458fd4d69-bgzfb

Connections
  id  host               container                                                      role    dir  security                                authentication  tenant  last dlv      uptime
  ===============================================================================================================================================================================================
  5   10.129.0.110:48498  bridge-3f5                                                    edge    in   no-security                             anonymous-user          000:00:00:02  000:17:36:29
  6   10.129.0.111:43254  rcv[default-cloud1-ceil-meter-smartgateway-58f885c76d-xmxwn]  edge    in   no-security                             anonymous-user          000:00:00:02  000:17:36:20
  7   10.130.0.109:50518  rcv[default-cloud1-coll-event-smartgateway-58fbbd4485-rl9bd]  normal  in   no-security                             anonymous-user          -             000:17:36:11
  8   10.130.0.110:33802  rcv[default-cloud1-ceil-event-smartgateway-6cfb65478c-g5q82]  normal  in   no-security                             anonymous-user          000:01:26:18  000:17:36:05
  22  10.128.0.1:51948   Router.ceph-0.redhat.local                                     edge    in   TLSv1/SSLv3(DHE-RSA-AES256-GCM-SHA384)  anonymous-user          000:00:00:03  000:22:08:43
  23  10.128.0.1:51950   Router.compute-0.redhat.local                                  edge    in   TLSv1/SSLv3(DHE-RSA-AES256-GCM-SHA384)  anonymous-user          000:00:00:03  000:22:08:43
  24  10.128.0.1:52082   Router.controller-0.redhat.local                               edge    in   TLSv1/SSLv3(DHE-RSA-AES256-GCM-SHA384)  anonymous-user          000:00:00:00  000:22:08:34
  27  127.0.0.1:42202    c2f541c1-4c97-4b37-a189-a396c08fb079                           normal  in   no-security                             no-auth                 000:00:00:00  000:00:00:00

$ oc exec -it deploy/default-interconnect -- qdstat --address

2020-04-21 18:20:10.293258 UTC
default-interconnect-7458fd4d69-bgzfb

Router Addresses
  class   addr                                phs  distrib    pri  local  remote  in           out          thru  fallback
  ==========================================================================================================================
  mobile  anycast/ceilometer/event.sample     0    balanced   -    1      0       970          970          0     0
  mobile  anycast/ceilometer/metering.sample  0    balanced   -    1      0       2,344,833    2,344,833    0     0
  mobile  collectd/notify                     0    multicast  -    1      0       70           70           0     0
  mobile  collectd/telemetry                  0    multicast  -    1      0       216,128,890  216,128,890  0     0

$ oc project service-telemetry

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  ...
spec:
  ...
  clouds:
  - name: cloud1
    events:
      collectors:
      - collectorType: collectd
        subscriptionAddress: collectd/cloud1-notify
      - collectorType: ceilometer
        subscriptionAddress: anycast/ceilometer/cloud1-event.sample
    metrics:
      collectors:
      - collectorType: collectd
        subscriptionAddress: collectd/cloud1-telemetry
      - collectorType: sensubility
        subscriptionAddress: sensubility/cloud1-telemetry
      - collectorType: ceilometer
        subscriptionAddress: anycast/ceilometer/cloud1-metering.sample
  - name: cloud2
    events:
      ...

$ oc get po -l app=smart-gateway
NAME                                                      READY   STATUS    RESTARTS   AGE
default-cloud1-ceil-event-smartgateway-6cfb65478c-g5q82   2/2     Running   0          13h
default-cloud1-ceil-meter-smartgateway-58f885c76d-xmxwn   2/2     Running   0          13h
default-cloud1-coll-event-smartgateway-58fbbd4485-rl9bd   2/2     Running   0          13h
default-cloud1-coll-meter-smartgateway-7c6fc495c4-jn728   2/2     Running   0          13h
default-cloud1-sens-meter-smartgateway-8h4tc445a2-mm683   2/2     Running   0          13h

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  ...
spec:
  ...
  cloudsRemoveOnMissing: true
  clouds:
    ...

$ oc get smartgateways

parameter_defaults:
    CloudDomain: newyork-west-04
    CephStorageHostnameFormat: 'ceph-%index%'
    ObjectStorageHostnameFormat: 'swift-%index%'
    ComputeHostnameFormat: 'compute-%index%'

resource_registry:
  OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml

parameter_defaults:
    ExtraConfig:
        qdr::router_id: %{::hostname}.cloud1

    MetricsQdrConnectors:
        - host: default-interconnect-5671-service-telemetry.apps.infra.watch
          port: 443
          role: edge
          verifyHostname: false
          sslProfile: sslProfile

    MetricsQdrSSLProfiles:
        - name: sslProfile
          caCertFileContent: |
            -----BEGIN CERTIFICATE-----
            <snip>
            -----END CERTIFICATE-----

    CeilometerQdrMetricsConfig:
        driver: amqp
        topic: cloud1-metering

    CollectdAmqpInstances:
        cloud1-telemetry:
            format: JSON
            presettle: false

    CollectdSensubilityResultsChannel: sensubility/cloud1-telemetry

$ source stackrc

(undercloud)$ openstack overcloud deploy --templates \
-e [your environment files] \
-e /usr/share/openstack-tripleo-heat-templates/environments/metrics/ceilometer-write-qdr.yaml \
-e /usr/share/openstack-tripleo-heat-templates/environments/metrics/qdr-edge-only.yaml \
-e /home/stack/hostnames.yaml \
-e /home/stack/enable-stf.yaml \
-e /home/stack/stf-connectors.yaml

$ oc apply -f - <<EOF
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  labels:
    operators.coreos.com/grafana-operator.openshift-operators: ""
  name: grafana-operator
  namespace: openshift-operators
spec:
  channel: v5
  installPlanApproval: Automatic
  name: grafana-operator
  source: community-operators
  sourceNamespace: openshift-marketplace
EOF

$ oc wait --for jsonpath="{.status.phase}"=Succeeded csv --namespace openshift-operators -l operators.coreos.com/grafana-operator.openshift-operators

clusterserviceversion.operators.coreos.com/grafana-operator.v5.6.0 condition met

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
...
spec:
  ...
  graphing:
    enabled: true
    grafana:
      ingressEnabled: true
      baseImage: 'registry.redhat.io/rhel8/grafana:9'

$ oc wait --for jsonpath="{.status.phase}"=Running pod -l app=default-grafana --timeout=600s

pod/default-grafana-deployment-669968df64-wz5s2 condition met

$ oc get grafanadatasources.grafana.integreatly.org

NAME                        NO MATCHING INSTANCES   LAST RESYNC   AGE
default-ds-stf-prometheus                           2m35s         2m56s

$ oc get route default-grafana-route

NAME                    HOST/PORT                                                   PATH   SERVICES                  PORT   TERMINATION   WILDCARD
default-grafana-route   default-grafana-route-service-telemetry.apps.infra.watch           default-grafana-service   web    reencrypt     None

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
...
spec:
  ...
  graphing:
    enabled: true
    grafana:
      dashboards:
        enabled: true

$ oc get grafanadashboards.grafana.integreatly.org

NAME                          NO MATCHING INSTANCES   LAST RESYNC   AGE
memcached-dashboard-1                                 38s           38s
rhos-cloud-dashboard-1                                39s           39s
rhos-dashboard-1                                      39s           39s
virtual-machine-dashboard-1                           37s           37s

$ oc get route default-grafana-route -ojsonpath='{.spec.host}'

default-grafana-route-service-telemetry.apps.infra.watch

$ oc project service-telemetry

$ oc create -f - <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: my-prometheus-reader-token
  namespace: service-telemetry
  annotations:
    kubernetes.io/service-account.name: stf-prometheus-reader
type: kubernetes.io/service-account-token
EOF

$ TOKEN=$(oc get secret my-prometheus-reader-token -o template='{{.data.token}}' | base64 -d)

$ PROM_HOST=$(oc get route default-prometheus-proxy -ogo-template='{{ .spec.host }}')

$ curl -k -H "Authorization: Bearer ${TOKEN}" https://${PROM_HOST}/api/v1/query?query=up

{"status":"success",[...]

$ echo $PROM_HOST
$ echo $TOKEN

$ oc delete secret my-prometheus-reader-token
secret "my-prometheus-reader-token" deleted

$ oc get po -l app.kubernetes.io/name=prometheus -w

$ oc describe po prometheus-default-0 | grep retention.time
      --storage.tsdb.retention.time=24h

$ oc project service-telemetry

$ curl -k -H "Authorization: Bearer $(oc create token stf-prometheus-reader)" https://$(oc get route default-prometheus-proxy -ogo-template='{{ .spec.host }}')/api/v1/rules

{"status":"success","data":{"groups":[{"name":"./openstack.rules","file":"/etc/prometheus/rules/prometheus-default-rulefiles-0/service-telemetry-prometheus-alarm-rules.yaml","rules":[{"state":"inactive","name":"Collectd metrics receive count is zero","query":"rate(sg_total_collectd_msg_received_count[1m]) == 0","duration":0,"labels":{},"annotations":{},"alerts":[],"health":"ok","evaluationTime":0.00034627,"lastEvaluation":"2021-12-07T17:23:22.160448028Z","type":"alerting"}],"interval":30,"evaluationTime":0.000353787,"lastEvaluation":"2021-12-07T17:23:22.160444017Z"}]}}

$ oc edit prometheusrules.monitoring.rhobs prometheus-alarm-rules

$ oc project service-telemetry

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
metadata:
  name: default
  namespace: service-telemetry
spec:
  backends:
    metrics:
      prometheus:
        enabled: true
  alertmanagerConfigManifest: |
    apiVersion: v1
    kind: Secret
    metadata:
      name: 'alertmanager-default'
      namespace: 'service-telemetry'
    type: Opaque
    stringData:
      alertmanager.yaml: |-
        global:
          resolve_timeout: 10m
        route:
          group_by: ['job']
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h
          receiver: 'null'
        receivers:
        - name: 'null'

$ oc get secret alertmanager-default -o go-template='{{index .data "alertmanager.yaml" | base64decode }}'

global:
  resolve_timeout: 10m
route:
  group_by: ['job']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'null'
receivers:
- name: 'null'

$ oc exec -it prometheus-default-0 -c prometheus -- sh -c "wget --header \"Authorization: Bearer \$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" https://default-alertmanager-proxy:9095/api/v1/status -q -O -"

{"status":"success","data":{"configYAML":"...",...}}

$ oc project service-telemetry

$ cat > alertmanager.yaml <<EOF
global:
  resolve_timeout: 10m
  slack_api_url: <slack_api_url>
receivers:
  - name: slack
    slack_configs:
    - channel: #stf-alerts
      title: |-
        ...
      text: >-
        ...
route:
  group_by: ['job']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'slack'
EOF

$ CONFIG_MANIFEST=$(oc create secret --dry-run=client generic alertmanager-default --from-file=alertmanager.yaml -o json)
$ oc patch stf default --type=merge -p '{"spec":{"alertmanagerConfigManifest":'"$CONFIG_MANIFEST"'}}'

$ oc get secret alertmanager-default -o go-template='{{index .data "alertmanager.yaml" | base64decode }}'

global:
  resolve_timeout: 10m
  slack_api_url: <slack_api_url>
receivers:
  - name: slack
    slack_configs:
    - channel: #stf-alerts
      title: |-
        ...
      text: >-
        ...
route:
  group_by: ['job']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'slack'

$ oc exec -it prometheus-default-0 -c prometheus -- /bin/sh -c "wget --header \"Authorization: Bearer \$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" https://default-alertmanager-proxy:9095/api/v1/status -q -O -"

{"status":"success","data":{"configYAML":"...",...}}

$ oc project service-telemetry

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
...
spec:
  ...
  alerting:
    alertmanager:
      receivers:
        snmpTraps:
          enabled: true
          target: 10.10.10.10

$ oc project service-telemetry

$ oc apply -f - <<EOF
apiVersion: monitoring.rhobs/v1
kind: PrometheusRule
metadata:
  creationTimestamp: null
  labels:
    prometheus: default
    role: alert-rules
  name: prometheus-alarm-rules-snmp
  namespace: service-telemetry
spec:
  groups:
    - name: ./openstack.rules
      rules:
        - alert: Collectd metrics receive rate is zero
          expr: rate(sg_total_collectd_msg_received_count[1m]) == 0
          labels:
            oid: 1.3.6.1.4.1.50495.15.1.2.1
            severity: critical
EOF

$ oc get secret  default-interconnect-selfsigned -o jsonpath='{.data.tls\.crt}' | base64 -d  | openssl x509 -in - -text | grep "Not After"
            Not After : Mar  9 21:00:16 2033 GMT

$ oc project service-telemetry

$ oc edit stf default

apiVersion: infra.watch/v1beta1
kind: ServiceTelemetry
...
spec:
  ...
  highAvailability:
    enabled: true

$ for o in alertmanagers.monitoring.rhobs/default prometheuses.monitoring.rhobs/default elasticsearch/elasticsearch grafana/default-grafana; do oc delete $o; done

$ oc get pods
NAME                                                      READY   STATUS    RESTARTS   AGE
default-cloud1-ceil-event-smartgateway-6f8547df6c-p2db5   3/3     Running   0          132m
default-cloud1-ceil-meter-smartgateway-59c845d65b-gzhcs   3/3     Running   0          132m
default-cloud1-coll-event-smartgateway-bf859f8d77-tzb66   3/3     Running   0          132m
default-cloud1-coll-meter-smartgateway-75bbd948b9-d5phm   3/3     Running   0          132m
default-cloud1-sens-meter-smartgateway-7fdbb57b6d-dh2g9   3/3     Running   0          132m
default-interconnect-668d5bbcd6-57b2l                     1/1     Running   0          132m
interconnect-operator-b8f5bb647-tlp5t                     1/1     Running   0          47h
service-telemetry-operator-566b9dd695-wkvjq               1/1     Running   0          156m
smart-gateway-operator-58d77dcf7-6xsq7                    1/1     Running   0          47h

$ oc edit namespace service-telemetry

metadata:
  labels:
    openshift.io/cluster-monitoring: "true"

$ for collector_type in ceil coll sens; do oc apply -f <(sed -e "s/<<COLLECTOR_TYPE>>/${collector_type}/g" << EOF

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    app: smart-gateway
  name: default-cloud1-<<COLLECTOR_TYPE>>-meter
  namespace: service-telemetry
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    interval: 10s
    metricRelabelings:
    - action: labeldrop
      regex: pod
    - action: labeldrop
      regex: namespace
    - action: labeldrop
      regex: instance
    - action: labeldrop
      regex: job
    - action: labeldrop
      regex: publisher
    port: prom-https
    scheme: https
    tlsConfig:
      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
      serverName: default-cloud1-<<COLLECTOR_TYPE>>-meter.service-telemetry.svc
  namespaceSelector:
    matchNames:
    - service-telemetry
  selector:
    matchLabels:
      app: smart-gateway
      smart-gateway: default-cloud1-<<COLLECTOR_TYPE>>-meter
EOF
); done
servicemonitor.monitoring.coreos.com/default-cloud1-ceil-meter configured
servicemonitor.monitoring.coreos.com/default-cloud1-coll-meter configured
servicemonitor.monitoring.coreos.com/default-cloud1-sens-meter configured

$ oc get route -n openshift-monitoring prometheus-k8s
NAME             HOST/PORT                                              PATH   SERVICES         PORT   TERMINATION          WILDCARD
prometheus-k8s   prometheus-k8s-openshift-monitoring.apps.infra.watch          prometheus-k8s   web    reencrypt/Redirect   None

  CollectdEnableLibpodstats: false

CollectdEnableSensubility: false