docker-android
docker-android copied to clipboard
budtmo
Unhealthy Container on Swarm
Note: Swarm nodes each have 19GB of memory
Summary: When running on a Docker Swarm the container cannot get healthy. The container starts it becomes unhealthy, gets killed then a new one starts and becomes unhealthy. It continues to loop like this
Steps to reproduce:
Terminal command:
docker stack deploy -c docker-compose.yml mobilev2Hub
docker-compose.yml:
version: '3.3'
services:
hub:
image: selenium/hub
ports:
- "4450:4444"
environment:
- GRID_BROWSER_TIMEOUT=60000
- GRID_TIMEOUT=90000
- GRID_MAX_SESSION=50
- GRID_CLEAN_UP_CYCLE=60000
- GRID_UNREGISTER_IF_STILL_DOWN_AFTER=180000
- GRID_NEW_SESSION_WAIT_TIMEOUT=120000
nexus:
image: butomo1989/docker-android-x86-7.1.1
depends_on:
- hub
ports:
- 6080
environment:
- DEVICE=Nexus 5
- CONNECT_TO_GRID=True
- APPIUM=true
- SELENIUM_HOST=selenium_hub
command: bash -c 'APPIUM_HOST=$$(hostname -i) /usr/bin/supervisord --configuration supervisord.conf'
Observed result: When the container starts it becomes unhealthy, gets killed then a new one starts and becomes unhealthy. It continues to loop like this
docker ps
docker service ps
Container Details
Robs-MacBook-Pro:docker-android rlupo$ docker container inspect 708f85118786
[
{
"Id": "708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e",
"Created": "2018-02-15T00:42:07.003481671Z",
"Path": "/bin/sh",
"Args": [
"-c",
"/usr/bin/supervisord --configuration supervisord.conf"
],
"State": {
"Status": "exited",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 0,
"ExitCode": 137,
"Error": "",
"StartedAt": "2018-02-15T00:42:09.426407663Z",
"FinishedAt": "2018-02-15T00:43:03.67098665Z",
"Health": {
"Status": "unhealthy",
"FailingStreak": 1,
"Log": [
{
"Start": "2018-02-15T00:42:11.428839526Z",
"End": "2018-02-15T00:42:51.429191943Z",
"ExitCode": -1,
"Output": "Health check exceeded timeout (40s)"
}
]
}
},
"Image": "sha256:5c95a383f428d1693a37973f74b134131b597ccc04d67913eb69b7ac3d43f859",
"ResolvConfPath": "/var/lib/docker/containers/708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e/hostname",
"HostsPath": "/var/lib/docker/containers/708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e/hosts",
"LogPath": "/var/lib/docker/containers/708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e/708f851187862e76646894a7eaed7b5d6e3f9e73202d46312c6a2c07d1a3cc5e-json.log",
"Node": {
"ID": "7LU7:UBLR:GFH6:MZTU:OZF2:FPJ4:BVRS:C2H4:GH2N:EL2J:2PS6:GQEX|10.65.190.54:12376",
"IP": "10.65.190.54",
"Addr": "10.65.190.54:12376",
"Name": "qa-docker04.avirat.net",
"Cpus": 8,
"Memory": 20877139968,
"Labels": {
"kernelversion": "3.10.0-693.11.1.el7.x86_64",
"network": "FW-OFWUTIL",
"operatingsystem": "CentOS Linux 7 (Core)",
"ostype": "linux",
"stage": "qa",
"storage": "nfs",
"storagedriver": "devicemapper"
}
},
"Name": "/mobilev2Hub_SamsungGalaxyS6.1.vrrkf2fz0zb1dpbdcj2q4ilgo",
"RestartCount": 0,
"Driver": "devicemapper",
"Platform": "",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Data": {
"DeviceId": "16211",
"DeviceName": "docker-253:2-100770711-59cc10b3c6c4d740d55b7c3ab0998cd6f579ef56a392b11a53d51d663173feda",
"DeviceSize": "32212254720"
},
"Name": "devicemapper"
},
"Mounts": [],
"Config": {
"Hostname": "708f85118786",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"4723/tcp": {},
"5554/tcp": {},
"5555/tcp": {},
"6080/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"APPIUM=True",
"CONNECT_TO_GRID=True",
"Device=\"Samsung Galaxy S6\"",
"SELENIUM_HOST=hub",
"SELENIUM_PORT=4450",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/bin:/root/tools:/root/tools/bin:/root/platform-tools:/root/build-tools:/root/build-tools",
"DEBIAN_FRONTEND=noninteractive",
"JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre",
"SDK_VERSION=sdk-tools-linux-3859397",
"ANDROID_BUILD_TOOLS_VERSION=27.0.2",
"ANDROID_HOME=/root",
"APPIUM_VERSION=1.7.2",
"TZ=US/Pacific",
"NOVNC_SHA=b403cb92fb8de82d04f305b4f14fa978003890d7",
"WEBSOCKIFY_SHA=558a6439f14b0d85a31145541745e25c255d576b",
"ANDROID_VERSION=7.0",
"API_LEVEL=24",
"PROCESSOR=x86",
"SYS_IMG=x86_64",
"IMG_TYPE=google_apis",
"BROWSER=chrome",
"LD_LIBRARY_PATH=/root/emulator/lib64:/root/emulator/lib64/qt/lib",
"DISPLAY=:0",
"SCREEN=0",
"SCREEN_WIDTH=1600",
"SCREEN_HEIGHT=900",
"SCREEN_DEPTH=16",
"LOCAL_PORT=5900",
"TARGET_PORT=6080",
"TIMEOUT=1",
"VIDEO_PATH=/tmp/video",
"LOG_PATH=/var/log/supervisor"
],
"Cmd": [
"/bin/sh",
"-c",
"/usr/bin/supervisord --configuration supervisord.conf"
],
"Healthcheck": {
"Test": [
"CMD-SHELL",
"adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done'"
],
"Interval": 2000000000,
"Timeout": 40000000000,
"Retries": 1
},
"ArgsEscaped": true,
"Image": "butomo1989/docker-android-x86-7.0:latest@sha256:74e79ae8cf2e7d126ba6982a70187b46648d2e331ba4d8296f0af34c1b294d78",
"Volumes": null,
"WorkingDir": "/root",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
"com.docker.stack.namespace": "mobilev2Hub",
"com.docker.swarm.node.id": "1sogr7twwvkqgrd4zbuv8eovn",
"com.docker.swarm.service.id": "p5vonbs86nyrg0pclijfoqosh",
"com.docker.swarm.service.name": "mobilev2Hub_SamsungGalaxyS6",
"com.docker.swarm.task": "",
"com.docker.swarm.task.id": "vrrkf2fz0zb1dpbdcj2q4ilgo",
"com.docker.swarm.task.name": "mobilev2Hub_SamsungGalaxyS6.1.vrrkf2fz0zb1dpbdcj2q4ilgo",
"com.docker.ucp.access.label": "qa-automation",
"com.docker.ucp.collection": "0ee9c208-fde2-428b-8f26-20e36a3d41ce",
"com.docker.ucp.collection.0ee9c208-fde2-428b-8f26-20e36a3d41ce": "true",
"com.docker.ucp.collection.c8c36d1e-bfa5-4d05-a29c-b66cd51a0d19": "true",
"com.docker.ucp.collection.root": "true",
"com.docker.ucp.collection.shared": "true",
"com.docker.ucp.collection.swarm": "true",
"maintainer": "Budi Utomo \[email protected]\u003e"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "e82f7bb7cc7acc75e4c93a3ee83303d96ae3aae3c6ce1651154d93028003f596",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/e82f7bb7cc7a",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"ingress": {
"IPAMConfig": {
"IPv4Address": "10.255.4.222"
},
"Links": null,
"Aliases": [
"708f85118786"
],
"NetworkID": "8jbcxo0ax5lhz6qxrwjk5whnd",
"EndpointID": "",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
},
"mobilev2Hub_default": {
"IPAMConfig": {
"IPv4Address": "10.0.14.5"
},
"Links": null,
"Aliases": [
"708f85118786"
],
"NetworkID": "hzqn2cduwapannbdmpk2s6meq",
"EndpointID": "",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
}
}
}
}
]
Hi @rlupo09 ,
Thank you for opening the issue. does it happen only on swarm?
@butomo1989 I'm facing the same problem.
I debugged it a bit more and this is what I see when executing docker exec -it <container_id> tail -f /var/log/supervisor/docker-android.stdout.log
emulator: CPU Acceleration: DISABLED
emulator: CPU Acceleration status: /dev/kvm is not found: VT disabled in BIOS or KVM kernel module not loaded
emulator: ERROR: x86 emulation currently requires hardware acceleration!
Please ensure KVM is properly installed and usable.
CPU acceleration status: /dev/kvm is not found: VT disabled in BIOS or KVM kernel module not loaded
The cause of this is that --privileged is not allowed in swarm mode so there's no access to /dev/kvm from inside of the container running in swarm mode.
Adding /dev/kvm:/dev/kvm volume doesn't help as there still won't be enough privileges set.
Any suggestions on how to work around this?
Hi @pawelus ,
Based on this article, it seems that docker will not add priviledge mode for docker swarm, but it shows how you can fork and modify it by yourself. Of course you need to pull new changes everyday if there is any from the original repo which isn't good.