cloud-on-k8s
cloud-on-k8s copied to clipboard
elastic
APM Server permissions for Kibana appears to be broken
When APM Server is deployed with ECK it has no sufficient permissions for Kibana API:
Transaction JSON
{
"_index": ".ds-traces-apm-epm_paas-2024.08.24-000002",
"_id": "x61jsZEB26iDE9PXHEpU",
"_version": 1,
"_score": 0,
"_source": {
"parent": {
"id": "154a123cf3b4f8ce"
},
"agent": {
"name": "go",
"version": "2.6.0"
},
"process": {
"args": [
"apm-server",
"run",
"-e",
"-c",
"config/config-secret/apm-server.yml"
],
"pid": 1,
"title": "apm-server"
},
"destination": {
"address": "kibana-v1-kb-http.apm-sandbox.svc",
"port": 5601
},
"processor": {
"event": "span"
},
"url": {
"original": "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
},
"labels": {
"project": "epm-paas"
},
"cloud": {
"availability_zone": "europe-west3-a",
"instance": {
"name": "gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn",
"id": "3933813931648332798"
},
"provider": "gcp",
"project": {
"id": "or2-ms-epm-iass-elastic-t1iylu"
},
"region": "europe-west3"
},
"observer": {
"hostname": "apm-server-v2-apm-server-56c7746446-m7dzp",
"type": "apm-server",
"version": "8.14.3"
},
"trace": {
"id": "154a123cf3b4f8ce0fb856d2d80a0416"
},
"@timestamp": "2024-09-02T06:19:00.318Z",
"data_stream": {
"namespace": "epm_paas",
"type": "traces",
"dataset": "apm"
},
"service": {
"node": {
"name": "apm-server-v2-apm-server-56c7746446-m7dzp"
},
"environment": "sandbox-latest",
"name": "apm-server",
"runtime": {
"name": "gc",
"version": "go1.22.5"
},
"language": {
"name": "go",
"version": "go1.22.5"
},
"version": "8.14.3",
"target": {
"name": "kibana-v1-kb-http.apm-sandbox.svc:5601",
"type": "http"
}
},
"host": {
"hostname": "apm-server-v2-apm-server-56c7746446-m7dzp",
"os": {
"platform": "linux"
},
"name": "apm-server-v2-apm-server-56c7746446-m7dzp",
"architecture": "amd64"
},
"http": {
"response": {
"status_code": 403
}
},
"event": {
"agent_id_status": "missing",
"ingested": "2024-09-02T06:19:08Z",
"success_count": 0,
"outcome": "failure"
},
"transaction": {
"id": "154a123cf3b4f8ce"
},
"span": {
"duration": {
"us": 102475
},
"representative_count": 1,
"stacktrace": [
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "span.go",
"line": {
"number": 442
},
"function": "(*Span).End",
"module": "go.elastic.co/apm/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 198
},
"function": "(*responseBody).endSpan",
"module": "go.elastic.co/apm/module/apmhttp/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 187
},
"function": "(*responseBody).Read",
"module": "go.elastic.co/apm/module/apmhttp/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 963
},
"function": "(*cancelTimerBody).Read",
"module": "net/http"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "io.go",
"line": {
"number": 712
},
"function": "ReadAll",
"module": "io"
},
{
"exclude_from_grouping": false,
"filename": "checkintegration.go",
"line": {
"number": 94
},
"function": "checkIntegrationInstalledKibana",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "checkintegration.go",
"line": {
"number": 57
},
"function": "checkIntegrationInstalled",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 629
},
"function": "(*Runner).waitReady.func3",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 638
},
"function": "(*Runner).waitReady.func4",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "waitready.go",
"line": {
"number": 59
},
"function": "waitReady",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 644
},
"function": "(*Runner).waitReady",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 331
},
"function": "(*Runner).Run.func4",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "errgroup.go",
"line": {
"number": 78
},
"function": "(*Group).Go.func1",
"module": "golang.org/x/sync/errgroup"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "asm_amd64.s",
"line": {
"number": 1695
},
"function": "goexit",
"module": "runtime"
}
],
"subtype": "http",
"destination": {
"service": {
"resource": "kibana-v1-kb-http.apm-sandbox.svc:5601",
"name": "https://kibana-v1-kb-http.apm-sandbox.svc:5601",
"type": "external"
}
},
"name": "GET kibana-v1-kb-http.apm-sandbox.svc:5601",
"id": "b9dd9b517374b4cf",
"type": "external"
},
"timestamp": {
"us": 1725257940318438
}
},
"fields": {
"host.hostname": [
"apm-server-v2-apm-server-56c7746446-m7dzp"
],
"url.original.text": [
"https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
],
"process.pid": [
1
],
"service.language.name": [
"go"
],
"cloud.availability_zone": [
"europe-west3-a"
],
"process.title.text": [
"apm-server"
],
"transaction.id": [
"154a123cf3b4f8ce"
],
"processor.event": [
"span"
],
"labels.project": [
"epm-paas"
],
"agent.name": [
"go"
],
"destination.address": [
"kibana-v1-kb-http.apm-sandbox.svc"
],
"host.name": [
"apm-server-v2-apm-server-56c7746446-m7dzp"
],
"event.agent_id_status": [
"missing"
],
"http.response.status_code": [
403
],
"event.outcome": [
"failure"
],
"cloud.region": [
"europe-west3"
],
"service.runtime.version": [
"go1.22.5"
],
"span.id": [
"b9dd9b517374b4cf"
],
"data_stream.type": [
"traces"
],
"span.type": [
"external"
],
"host.architecture": [
"amd64"
],
"cloud.provider": [
"gcp"
],
"timestamp.us": [
1725257940318438
],
"observer.type": [
"apm-server"
],
"observer.version": [
"8.14.3"
],
"agent.version": [
"2.6.0"
],
"parent.id": [
"154a123cf3b4f8ce"
],
"span.destination.service.name": [
"https://kibana-v1-kb-http.apm-sandbox.svc:5601"
],
"process.title": [
"apm-server"
],
"span.representative_count": [
1
],
"span.destination.service.type": [
"external"
],
"span.name": [
"GET kibana-v1-kb-http.apm-sandbox.svc:5601"
],
"destination.port": [
5601
],
"service.node.name": [
"apm-server-v2-apm-server-56c7746446-m7dzp"
],
"cloud.instance.id": [
"3933813931648332798"
],
"trace.id": [
"154a123cf3b4f8ce0fb856d2d80a0416"
],
"span.duration.us": [
102475
],
"span.stacktrace": [
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "span.go",
"line": {
"number": 442
},
"function": "(*Span).End",
"module": "go.elastic.co/apm/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 198
},
"function": "(*responseBody).endSpan",
"module": "go.elastic.co/apm/module/apmhttp/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 187
},
"function": "(*responseBody).Read",
"module": "go.elastic.co/apm/module/apmhttp/v2"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "client.go",
"line": {
"number": 963
},
"function": "(*cancelTimerBody).Read",
"module": "net/http"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "io.go",
"line": {
"number": 712
},
"function": "ReadAll",
"module": "io"
},
{
"exclude_from_grouping": false,
"filename": "checkintegration.go",
"line": {
"number": 94
},
"function": "checkIntegrationInstalledKibana",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "checkintegration.go",
"line": {
"number": 57
},
"function": "checkIntegrationInstalled",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 629
},
"function": "(*Runner).waitReady.func3",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 638
},
"function": "(*Runner).waitReady.func4",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "waitready.go",
"line": {
"number": 59
},
"function": "waitReady",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 644
},
"function": "(*Runner).waitReady",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "beater.go",
"line": {
"number": 331
},
"function": "(*Runner).Run.func4",
"module": "github.com/elastic/apm-server/internal/beater"
},
{
"exclude_from_grouping": false,
"filename": "errgroup.go",
"line": {
"number": 78
},
"function": "(*Group).Go.func1",
"module": "golang.org/x/sync/errgroup"
},
{
"exclude_from_grouping": false,
"library_frame": true,
"filename": "asm_amd64.s",
"line": {
"number": 1695
},
"function": "goexit",
"module": "runtime"
}
],
"event.success_count": [
0
],
"service.target.type": [
"http"
],
"service.environment": [
"sandbox-latest"
],
"service.name": [
"apm-server"
],
"data_stream.namespace": [
"epm_paas"
],
"service.runtime.name": [
"gc"
],
"process.args": [
"apm-server",
"run",
"-e",
"-c",
"config/config-secret/apm-server.yml"
],
"span.subtype": [
"http"
],
"service.target.name": [
"kibana-v1-kb-http.apm-sandbox.svc:5601"
],
"observer.hostname": [
"apm-server-v2-apm-server-56c7746446-m7dzp"
],
"event.ingested": [
"2024-09-02T06:19:08.000Z"
],
"url.original": [
"https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm"
],
"@timestamp": [
"2024-09-02T06:19:00.318Z"
],
"service.version": [
"8.14.3"
],
"host.os.platform": [
"linux"
],
"data_stream.dataset": [
"apm"
],
"service.language.version": [
"go1.22.5"
],
"span.destination.service.resource": [
"kibana-v1-kb-http.apm-sandbox.svc:5601"
],
"cloud.instance.name": [
"gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn"
],
"cloud.project.id": [
"or2-ms-epm-iass-elastic-t1iylu"
]
}
}
Could you please provide the manifests you are using, this would help me reproduce. Thanks!
Could you please provide the manifests you are using, this would help me reproduce. Thanks!
It's slightly sanitized from sensitive data: affinity, specific labels, but cross-resource refers are persisted "as-is"
apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
labels:
module/name: apm-server
package/name: intake
package/overlay: base
package/version: "2"
name: apm-server-v2
namespace: apm-sandbox
spec:
config:
apm-server:
auth:
anonymous:
allow_agent:
- rum-js
- rum-js-dpeo
- js-base
- java
- dotnet
- php
- opentelemetry/cpp
- python
- otlp
- go
- opentelemetry
- opentelemetry/webjs
- opentelemetry/js
- opentelemetry/go
- opentelemetry/java
- opentelemetry/nodejs
- opentelemetry/dotnet
- nodejs
- '@microlabs/otel-workers-sdk/js'
enabled: true
rate_limit:
event_limit: 8000
ip_limit: 1000
api_key:
enabled: false
limit: 100
capture_personal_data: true
default_service_environment: undefined
expvar.enabled: false
host: 0.0.0.0:8200
idle_timeout: 45s
max_connections: 0
max_event_size: 307200
max_header_size: 1048576
pprof.enabled: false
read_timeout: 30s
rum:
allow_headers:
- x-requested-with
- access-control-request-private-network
- access-control-allow-origin
- xmlhttprequest
- request-origin
allow_origins:
- '*'
enabled: true
exclude_from_grouping: ^/webpack
library_pattern: node_modules|bower_components|~
shutdown_timeout: 30s
ssl:
supported_protocols:
- TLSv1.2
- TLSv1.3
write_timeout: 30s
logging.level: warning
monitoring.elasticsearch: {}
count: 2
elasticsearchRef:
name: elasticsearch-v1
http:
service:
metadata:
labels:
module/name: apm-server
package/name: intake
package/version: "2"
spec:
ports:
- appProtocol: HTTPS
name: https
port: 8200
protocol: TCP
targetPort: 8200
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: apm-server
kibanaRef:
name: kibana-v1
podTemplate:
metadata:
creationTimestamp: null
labels:
module/name: apm-server
package/name: intake
package/version: "2"
spec:
containers:
- env:
- name: ELASTIC_APM_GLOBAL_LABELS
value: project=dummy
- name: ELASTIC_APM_CAPTURE_BODY
value: all
- name: ELASTICSEARCH_HOST
value: https://elasticsearch:9200
name: apm-server
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 1
memory: 1Gi
topologySpreadConstraints:
- labelSelector:
matchLabels:
module/name: apm-server
package/name: intake
package/version: "2"
maxSkew: 1
nodeAffinityPolicy: Honor
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
version: 8.14.3
---
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
name: kibana-v1
namespace: apm-sandbox
spec:
config:
elasticsearch.requestHeadersWhitelist:
- authorization
elasticsearch.requestTimeout: 60000
elasticsearch.shardTimeout: 60000
server:
customResponseHeaders:
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-XSS-Protection: 1; mode=block
telemetry.optIn: false
xpack.fleet.agentPolicies:
- id: eck-fleet-server
monitoring_enabled:
- logs
- metrics
name: Fleet Server on ECK policy
namespace: default
package_policies:
- id: fleet_server-1
name: fleet_server-1
package:
name: fleet_server
unenroll_timeout: 900
xpack.fleet.agents.fleet_server.hosts:
- https://fleet:8220
xpack.fleet.outputs:
- config:
ssl.verification_mode: none
hosts:
- https://elasticsearch:9200
id: fleet-default-output
is_default: "true"
is_default_monitoring: "true"
name: default
type: elasticsearch
xpack.fleet.packages:
- name: system
version: latest
- name: elastic_agent
version: latest
- name: fleet_server
version: latest
- name: apm
version: latest
- name: kubernetes
version: latest
- name: cloudflare
version: latest
- name: synthetics
version: latest
- name: cloudflare_logpush
version: latest
- name: gcp_pubsub
version: latest
xpack.reporting.roles.enabled: false
xpack.spaces.maxSpaces: 1000
xpack.task_manager.max_workers: 100
xpack.task_manager.monitored_stats_health_verbose_log.enabled: true
count: 3
elasticsearchRef:
name: elasticsearch-v1
enterpriseSearchRef: {}
http:
service:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
- name: https
port: 5601
protocol: TCP
targetPort: 5601
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: kibana
monitoring:
logs: {}
metrics: {}
podTemplate:
metadata:
creationTimestamp: null
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
common.k8s.elastic.co/type: kibana
package/name: elastic-stack-eck
package/type: component
package/version: "1"
topologyKey: kubernetes.io/hostname
containers:
- name: kibana
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
version: 8.14.3
---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
name: elasticsearch-v1
spec:
auth: {}
http:
service:
metadata: {}
spec: {}
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: elasticsearch
monitoring:
logs: {}
metrics: {}
nodeSets:
- config:
node.roles:
- master
- remote_cluster_client
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 3
name: master
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 2
memory: 10Gi
requests:
cpu: 1
memory: 10Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: standard-rwo
- config:
node.roles:
- remote_cluster_client
- data_content
- data_hot
- ingest
- transform
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 4
name: data
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 7
memory: 54Gi
requests:
cpu: 6
memory: 54Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Ti
storageClassName: premium-rwo
- config:
node.roles:
- remote_cluster_client
- data_cold
- data_warm
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 3
name: data-cold
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 4
memory: 16Gi
requests:
cpu: 2
memory: 16Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3096Gi
storageClassName: standard-rwo
podDisruptionBudget:
metadata: {}
spec:
maxUnavailable: 1
selector:
matchLabels:
common.k8s.elastic.co/type: elasticsearch
package/name: elastic-stack-eck
package/type: component
package/version: "1"
transport:
service:
metadata: {}
spec: {}
tls:
certificate: {}
certificateAuthorities: {}
updateStrategy:
changeBudget:
maxUnavailable: 1
version: 8.14.3
I'm sorry I've not been able to reproduce (maybe because of the changes I had to make to your manifest). I'm not entirely sure it is an orchestration issue I would suggest to start a discussion in https://discuss.elastic.co/c/observability/apm/58
If it turns out something is missing on the ECK side please feel free to open a new issue.