toolkit
toolkit copied to clipboard
overleaf
Mongo instance doesn't support transactions when upgrading to 4.x.x
Hey, I'm in the process of upgrading my services. I had overleaf ~3.2 running and upgraded it to 3.5.13 + ran the history migration and cleanup scripts.
When changing config/version to 4.0.1 or 4.0.6 I get the following error log:
mongo | 2023-10-21T12:48:00.794+0000 I NETWORK [conn10] received client metadata from 172.28.0.6:33180 conn10: { driver: { name: "nodejs|Mongoose", version: "4.13.0" }, os: { type: "Linux", name: "linux", architecture: "x64", version: "6.1.0-7-amd64" }, platform: "Node.js v16.20.1, LE (unified)", version: "4.13.0|6.9.1", application: { name: "web" } }
sharelatex | Mongo instance doesn't support transactions
sharelatex | MongoServerError: Transactions are only supported in featureCompatibilityVersion 4.0. See http://dochub.mongodb.org/core/4.0-feature-compatibility for more information.
sharelatex | at Connection.onMessage (/overleaf/services/web/node_modules/mongodb/lib/cmap/connection.js:227:30)
sharelatex | at MessageStream.<anonymous> (/overleaf/services/web/node_modules/mongodb/lib/cmap/connection.js:60:60)
sharelatex | at MessageStream.emit (node:events:513:28)
sharelatex | at processIncomingData (/overleaf/services/web/node_modules/mongodb/lib/cmap/message_stream.js:125:16)
sharelatex | at MessageStream._write (/overleaf/services/web/node_modules/mongodb/lib/cmap/message_stream.js:33:9)
sharelatex | at writeOrBuffer (node:internal/streams/writable:391:12)
sharelatex | at _write (node:internal/streams/writable:332:10)
sharelatex | at MessageStream.Writable.write (node:internal/streams/writable:336:10)
sharelatex | at Socket.ondata (node:internal/streams/readable:754:22)
sharelatex | at Socket.emit (node:events:513:28) {
sharelatex | operationTime: new Timestamp({ t: 1697892477, i: 1 }),
sharelatex | ok: 0,
sharelatex | code: 50773,
sharelatex | codeName: 'Location50773',
sharelatex | '$clusterTime': {
sharelatex | clusterTime: new Timestamp({ t: 1697892477, i: 1 }),
sharelatex | signature: {
sharelatex | hash: new Binary(Buffer.from("0000000000000000000000000000000000000000", "hex"), 0),
sharelatex | keyId: 0
sharelatex | }
sharelatex | },
sharelatex | [Symbol(errorLabels)]: Set(0) {}
sharelatex | }
mongo | 2023-10-21T12:48:00.993+0000 I NETWORK [conn10] end connection 172.28.0.6:33180 (3 connections now open)
This post suggests that the wiredTrigger engine and replica sets need to be used, both seem to be the case as docker logs mongo suggests:
2023-10-21T12:47:27.991+0000 I CONTROL [initandlisten] options: { net: { bindIpAll: true }, replication: { replSet: "overleaf" } }
2023-10-21T12:47:27.994+0000 I STORAGE [initandlisten] Detected data files in /data/db created by the 'wiredTiger' storage engine, so setting the active storage engine to 'wiredTiger'.
2023-10-21T12:47:27.994+0000 I STORAGE [initandlisten]
2023-10-21T12:47:27.994+0000 I STORAGE [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
2023-10-21T12:47:27.994+0000 I STORAGE [initandlisten] ** See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-10-21T12:47:27.994+0000 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=3457M,cache_overflow=(file_max=0M),session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),statistics_log=(wait=0),verbose=(recovery_progress),
2023-10-21T12:47:33.404+0000 I STORAGE [initandlisten] WiredTiger message [1697892453:404951][1:0x7fb7d18a3a80], txn-recover: Main recovery loop: starting at 2596/896 to 2597/256
2023-10-21T12:47:33.891+0000 I STORAGE [initandlisten] WiredTiger message [1697892453:891854][1:0x7fb7d18a3a80], txn-recover: Recovering log 2596 through 2597
2023-10-21T12:47:34.293+0000 I STORAGE [initandlisten] WiredTiger message [1697892454:293811][1:0x7fb7d18a3a80], txn-recover: Recovering log 2597 through 2597
2023-10-21T12:47:34.895+0000 I STORAGE [initandlisten] WiredTiger message [1697892454:895658][1:0x7fb7d18a3a80], txn-recover: Set global recovery timestamp: 0
2023-10-21T12:47:35.302+0000 I RECOVERY [initandlisten] WiredTiger recoveryTimestamp. Ts: Timestamp(0, 0)
Running rs.initiate({ _id: "overleaf", members: [ { _id: 0, host: "mongo:27017" } ] }) as suggested in #165 just tells me that the replica set already exists
overleaf:PRIMARY> rs.initiate({ _id: "overleaf", members: [ { _id: 0, host: "mongo:27017" } ] })
{
"operationTime" : Timestamp(1697892737, 1),
"ok" : 0,
"errmsg" : "already initialized",
"code" : 23,
"codeName" : "AlreadyInitialized",
"$clusterTime" : {
"clusterTime" : Timestamp(1697892737, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
I tried t0:
- shut all containers down, delete them with
docker system prune -aand recreate them --> same issue - delete my
datadirectory and replace it with the backup from before upgrading. Ran the upgrades again and had the same issue
Downgrading to 3.5.13 for now as this version works
If it helps here is a redacted version of docker inspect mongo
[
{
"Id": "redacted",
"Created": "2023-10-21T12:47:23.158280168Z",
"Path": "docker-entrypoint.sh",
"Args": [
"--replSet",
"overleaf"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 459954,
"ExitCode": 0,
"Error": "",
"StartedAt": "2023-10-21T12:47:27.293435143Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Health": {
"Status": "healthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2023-10-21T15:02:54.725794691+02:00",
"End": "2023-10-21T15:02:56.348036238+02:00",
"ExitCode": 0,
"Output": "1\n"
},
{
"Start": "2023-10-21T15:03:06.676475861+02:00",
"End": "2023-10-21T15:03:08.660895838+02:00",
"ExitCode": 0,
"Output": "1\n"
},
{
"Start": "2023-10-21T15:03:18.714861231+02:00",
"End": "2023-10-21T15:03:19.68932508+02:00",
"ExitCode": 0,
"Output": "1\n"
},
{
"Start": "2023-10-21T15:03:29.765612028+02:00",
"End": "2023-10-21T15:03:31.269786405+02:00",
"ExitCode": 0,
"Output": "1\n"
},
{
"Start": "2023-10-21T15:03:41.297791966+02:00",
"End": "2023-10-21T15:03:42.433200087+02:00",
"ExitCode": 0,
"Output": "1\n"
}
]
}
},
"Image": "sha256:fb1435e8841ce13f9ef0d245fe2ab73730d23f33f99cc43b3754248d36e70ae7",
"ResolvConfPath": "/var/lib/docker/containers/redacted/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/redacted/hostname",
"HostsPath": "/var/lib/docker/containers/redacted/hosts",
"LogPath": "/var/lib/docker/containers/redacted/redacted-json.log",
"Name": "/mongo",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"redacted/overleaf/data/mongo:/data/db:rw"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "5",
"max-size": "8m"
}
},
"NetworkMode": "web",
"PortBindings": {},
"RestartPolicy": {
"Name": "always",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": [],
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"Mounts": [
{
"Type": "volume",
"Source": "redacted",
"Target": "/data/configdb"
}
],
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/redacted-init/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff:/var/lib/docker/overlay2/redacted/diff",
"MergedDir": "/var/lib/docker/overlay2/redacted/merged",
"UpperDir": "/var/lib/docker/overlay2/redacted/diff",
"WorkDir": "/var/lib/docker/overlay2/redacted/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "redacted/overleaf/data/mongo",
"Destination": "/data/db",
"Mode": "rw",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "redacted",
"Source": "/var/lib/docker/volumes/redacted/_data",
"Destination": "/data/configdb",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "redacted",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": true,
"AttachStderr": true,
"ExposedPorts": {
"27017": {},
"27017/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"GOSU_VERSION=1.12",
"JSYAML_VERSION=3.13.1",
"MONGO_PACKAGE=mongodb-org",
"MONGO_REPO=repo.mongodb.org",
"MONGO_MAJOR=4.0",
"MONGO_VERSION=4.0.28",
"HOME=/data/db"
],
"Cmd": [
"--replSet",
"overleaf"
],
"Healthcheck": {
"Test": [
"CMD-SHELL",
"echo 'db.stats().ok' | mongo localhost:27017/test --quiet"
],
"Interval": 10000000000,
"Timeout": 10000000000,
"Retries": 5
},
"Image": "mongo:4.0",
"Volumes": {
"/data/configdb": {},
"/data/db": {}
},
"WorkingDir": "",
"Entrypoint": [
"docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"com.docker.compose.config-hash": "e3b1d5ca67e7f95fdbddc2ff1480d06dcc9582a80aa5a18645641a7737af771b",
"com.docker.compose.container-number": "1",
"com.docker.compose.depends_on": "",
"com.docker.compose.image": "sha256:fb1435e8841ce13f9ef0d245fe2ab73730d23f33f99cc43b3754248d36e70ae7",
"com.docker.compose.oneoff": "False",
"com.docker.compose.project": "overleaf",
"com.docker.compose.project.config_files": "redacted/overleaf/lib/docker-compose.base.yml,redacted/overleaf/lib/docker-compose.redis.yml,redacted/overleaf/lib/docker-compose.mongo.yml",
"com.docker.compose.project.working_dir": "redacted/overleaf/lib",
"com.docker.compose.replace": "redacted",
"com.docker.compose.service": "mongo",
"com.docker.compose.version": "2.17.2"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "redacted",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"27017/tcp": null
},
"SandboxKey": "/var/run/docker/netns/redacted",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"web": {
"IPAMConfig": null,
"Links": null,
"Aliases": [
"mongo",
"mongo",
"redacted"
],
"NetworkID": "redacted",
"EndpointID": "redacted",
"Gateway": "172.28.0.1",
"IPAddress": "172.28.0.4",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "redacted",
"DriverOpts": null
}
}
}
}
]
Thanks for the detailed issue. I ran into the same error and version 3.5.13 saved the day. It would be nice to see a resolution here.
