tinygo
tinygo copied to clipboard
tinygo-org
memory leak
The following program never frees memory, even with -gc=conservative.
Build it via tinygo build -scheduler=none foo.go
package main
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"log"
"runtime"
)
func logError(err error) {
log.Print(err)
}
func wrap(message string, err error) error {
return errors.Join(errors.New(message), err)
}
func marshall(data map[string]any) ([]byte, error) {
b, err := json.Marshal(data)
if err != nil {
return []byte{}, err
}
return b, nil
}
func unmarshall(bytes []byte) (map[string]any, error) {
var data map[string]any
err := json.Unmarshal(bytes, &data)
if err != nil {
return make(map[string]any), err
}
return data, nil
}
func redact(bytes []byte) ([]byte, error) {
data, err := unmarshall(bytes)
if err != nil {
return []byte{}, wrap("unable to unmarshall record", err)
}
bytes, err = marshall(data)
if err != nil {
return []byte{}, wrap("unable to marshall record", err)
}
return bytes, nil
}
func main() {
var m runtime.MemStats
data := []byte("{\"version\":0,\"id\":\"a0fc7164-3e81-4b0a-b806-ae588be18148\",\"createdAt\":\"2024-01-03T10:05:27.182320941Z\",\"lastUpdatedAt\":\"2024-01-03T10:05:27.182320983Z\",\"deliveredAt\":null,\"completedAt\":null,\"customer\":{\"version\":0,\"id\":\"4f63ef1e-4933-4355-b6ed-341bf462ae97\",\"firstName\
":\"Troy\",\"lastName\":\"Ledner\",\"gender\":\"male\",\"companyName\":null,\"email\":\"[email protected]\",\"customerType\":\"PERSONAL\",\"revision\":0},\"orderValue\":208202,\"lineItems\":[{\"articleId\":\"bdead2a9-63de-4265-961b-3d2827f95ed5\",\"name\":\"Carrot\",\"quantity\":394,\"quantityUnit\":\"gram\",\"
unitPrice\":548,\"totalPrice\":215912},{\"articleId\":\"0a1bea03-2374-4b69-9f69-73cc2ece13bc\",\"name\":\"Cucumber\",\"quantity\":498,\"quantityUnit\":\"pieces\",\"unitPrice\":694,\"totalPrice\":345612},{\"articleId\":\"ecc32ad4-f5ba-4a94-b18a-94af9b322fa6\",\"name\":\"Corn\",\"quantity\":124,\"quantityUnit\":\"gram\"
,\"unitPrice\":685,\"totalPrice\":84940},{\"articleId\":\"f3199e80-afde-4de6-9e0f-982ff4097efd\",\"name\":\"Beans, Green\",\"quantity\":83,\"quantityUnit\":\"gram\",\"unitPrice\":342,\"totalPrice\":28386},{\"articleId\":\"4495e816-b5d5-44e1-b1cc-340f9850c0d8\",\"name\":\"Lettuce\",\"quantity\":2,\"quantityUnit\":\"pie
ces\",\"unitPrice\":790,\"totalPrice\":1580},{\"articleId\":\"83825065-c7a3-4174-885d-671917018722\",\"name\":\"Amaranth Leaves\",\"quantity\":190,\"quantityUnit\":\"pieces\",\"unitPrice\":987,\"totalPrice\":187530},{\"articleId\":\"fe140610-83e0-46a9-b2c6-6efe727d4a6f\",\"name\":\"Soybeans\",\"quantity\":179,\"quanti
tyUnit\":\"pieces\",\"unitPrice\":503,\"totalPrice\":90037},{\"articleId\":\"df8107b0-eeda-46a1-98c8-8a75d9ff052e\",\"name\":\"Fennel\",\"quantity\":271,\"quantityUnit\":\"gram\",\"unitPrice\":135,\"totalPrice\":36585},{\"articleId\":\"482ef729-3160-459a-98db-2a607a5f51d0\",\"name\":\"Fiddleheads\",\"quantity\":498,\"
quantityUnit\":\"gram\",\"unitPrice\":636,\"totalPrice\":316728},{\"articleId\":\"500ec523-daa7-43fc-badc-87c754ba5b2a\",\"name\":\"Celeriac\",\"quantity\":94,\"quantityUnit\":\"pieces\",\"unitPrice\":6,\"totalPrice\":564},{\"articleId\":\"af8207f7-e18d-4f39-bf98-bf5f1a0cf985\",\"name\":\"Spaghetti Squash\",\"quantity
\":208,\"quantityUnit\":\"pieces\",\"unitPrice\":83,\"totalPrice\":17264},{\"articleId\":\"fc95b378-ef2c-4048-8817-143c636b32fd\",\"name\":\"Zucchini\",\"quantity\":250,\"quantityUnit\":\"pieces\",\"unitPrice\":878,\"totalPrice\":219500},{\"articleId\":\"3a7a8c03-ee17-43e1-84a7-4f5bea5adfb6\",\"name\":\"Kohlrabi\",\"q
uantity\":238,\"quantityUnit\":\"pieces\",\"unitPrice\":941,\"totalPrice\":223958},{\"articleId\":\"0e69b9c4-ffb9-4d32-8897-a21c0508c6ba\",\"name\":\"Fiddleheads\",\"quantity\":190,\"quantityUnit\":\"gram\",\"unitPrice\":763,\"totalPrice\":144970},{\"articleId\":\"d4112bf9-b681-4481-a11f-13564d82010c\",\"name\":\"Arro
wroot\",\"quantity\":45,\"quantityUnit\":\"gram\",\"unitPrice\":979,\"totalPrice\":44055},{\"articleId\":\"6c707870-1825-490f-8c10-5ed3350ce6f2\",\"name\":\"Amaranth Leaves\",\"quantity\":261,\"quantityUnit\":\"gram\",\"unitPrice\":876,\"totalPrice\":228636},{\"articleId\":\"085d0bf0-8320-4670-83a6-fb35d449332d\",\"na
me\":\"Chicory\",\"quantity\":197,\"quantityUnit\":\"gram\",\"unitPrice\":322,\"totalPrice\":63434},{\"articleId\":\"047ffb3e-259c-4b5d-bead-f491b136de4e\",\"name\":\"Crookneck\",\"quantity\":216,\"quantityUnit\":\"gram\",\"unitPrice\":49,\"totalPrice\":10584},{\"articleId\":\"2a198c53-2193-46b3-8451-185bae4f641c\",\"
name\":\"Corn\",\"quantity\":492,\"quantityUnit\":\"gram\",\"unitPrice\":346,\"totalPrice\":170232},{\"articleId\":\"5f171075-62b8-4ad9-ac8a-4fe7bce8ee6a\",\"name\":\"Kohlrabi\",\"quantity\":86,\"quantityUnit\":\"gram\",\"unitPrice\":79,\"totalPrice\":6794},{\"articleId\":\"a225608d-4dba-4be2-b0a9-e4cf1f29bc68\",\"nam
e\":\"Spaghetti Squash\",\"quantity\":190,\"quantityUnit\":\"gram\",\"unitPrice\":818,\"totalPrice\":155420},{\"articleId\":\"76defe1a-708d-4345-b078-e3f5ad35c5c7\",\"name\":\"Crookneck\",\"quantity\":452,\"quantityUnit\":\"pieces\",\"unitPrice\":686,\"totalPrice\":310072},{\"articleId\":\"d1a12461-5214-4cb5-a8b0-8332
9735bcd8\",\"name\":\"Cauliflower\",\"quantity\":269,\"quantityUnit\":\"pieces\",\"unitPrice\":273,\"totalPrice\":73437},{\"articleId\":\"5fb1278e-d4f5-4307-879a-c5c326dc9ecc\",\"name\":\"Chicory\",\"quantity\":309,\"quantityUnit\":\"pieces\",\"unitPrice\":559,\"totalPrice\":172731},{\"articleId\":\"06b1b083-4ec3-46e3
-8752-ed36d2b92f94\",\"name\":\"Artichoke\",\"quantity\":148,\"quantityUnit\":\"gram\",\"unitPrice\":959,\"totalPrice\":141932}],\"payment\":{\"paymentId\":\"c54a61d2-b371-4146-bd69-009b0db3dbb0\",\"method\":\"DEBIT\"},\"deliveryAddress\":{\"version\":0,\"id\":\"7bdb0bb5-de09-443e-b42d-bc8ad1abce75\",\"customer\":{\"i
d\":\"4f63ef1e-4933-4355-b6ed-341bf462ae97\",\"type\":\"PERSONAL\"},\"type\":\"INVOICE\",\"firstName\":\"Troy\",\"lastName\":\"Ledner\",\"state\":\"South Carolina\",\"street\":\"31494 Rapids haven\",\"houseNumber\":\"762\",\"city\":\"Melissamouth\",\"zip\":\"99100\",\"latitude\":-4.672282,\"longitude\":68.182552,\"pho
ne\":\"599.533.2979\",\"additionalAddressInfo\":\"\",\"createdAt\":\"2024-01-03T10:05:27.272445772Z\",\"revision\":0},\"revision\":0}")
for i := 0; i < 1000000; i++ {
redacted, err := redact(data)
if err != nil {
panic(wrap("unable to redact record", err))
}
eq := bytes.Equal(redacted, redacted)
if !eq {
panic("oops")
}
if i%1000 == 0 {
runtime.ReadMemStats(&m)
fmt.Printf("%+v\n", &m)
}
}
}
BigGo seems to handle freeing memory fine based on the ReadMemStats call, but in TinyGo the memory just grows unbounded until the program exits or OOMs.
An easy way to test is via wasmtime and capping memory usage:
tinygo build -scheduler=none -target=wasi foo.go
wasmtime -W max-memory-size=8000000 -W max-table-elements=100 foo.wasm
Here's the output of running it with wasi and wasmtime on my machine
% wasmtime -W max-memory-size=8000000 -W max-table-elements=100 transform.wasm
&{Sys:146768 HeapSys:144496 HeapIdle:35424 HeapInuse:109072 HeapReleased:0 TotalAlloc:93378 Mallocs:2244 Frees:133 GCSys:2258}
&{Sys:1981776 HeapSys:1951280 HeapIdle:1005872 HeapInuse:945408 HeapReleased:0 TotalAlloc:54923257 Mallocs:2150708 Frees:2137736 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:948144 HeapInuse:1003136 HeapReleased:0 TotalAlloc:109757189 Mallocs:4299194 Frees:4274099 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:1499888 HeapInuse:451392 HeapReleased:0 TotalAlloc:164587505 Mallocs:6447655 Frees:6441040 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:1058960 HeapInuse:892320 HeapReleased:0 TotalAlloc:219413061 Mallocs:8596082 Frees:8582625 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:361184 HeapInuse:1590096 HeapReleased:0 TotalAlloc:274241417 Mallocs:10744529 Frees:10709940 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:1112544 HeapInuse:838736 HeapReleased:0 TotalAlloc:329074253 Mallocs:12893008 Frees:12878153 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:387472 HeapInuse:1563808 HeapReleased:0 TotalAlloc:383903869 Mallocs:15041464 Frees:15006510 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:640496 HeapInuse:1310784 HeapReleased:0 TotalAlloc:438736985 Mallocs:17189945 Frees:17154942 GCSys:30489}
&{Sys:1981776 HeapSys:1951280 HeapIdle:559632 HeapInuse:1391648 HeapReleased:0 TotalAlloc:493565341 Mallocs:19338392 Frees:19308674 GCSys:30489}
&{Sys:4078928 HeapSys:4016160 HeapIdle:909984 HeapInuse:3106176 HeapReleased:0 TotalAlloc:548393837 Mallocs:21486840 Frees:21420174 GCSys:62753}
&{Sys:4078928 HeapSys:4016160 HeapIdle:708016 HeapInuse:3308144 HeapReleased:0 TotalAlloc:603223873 Mallocs:23635299 Frees:23563001 GCSys:62753}
&{Sys:4078928 HeapSys:4016160 HeapIdle:2396176 HeapInuse:1619984 HeapReleased:0 TotalAlloc:658054189 Mallocs:25783760 Frees:25755534 GCSys:62753}
&{Sys:4078928 HeapSys:4016160 HeapIdle:2902768 HeapInuse:1113392 HeapReleased:0 TotalAlloc:712885485 Mallocs:27932228 Frees:27918987 GCSys:62753}
&{Sys:4078928 HeapSys:4016160 HeapIdle:1072864 HeapInuse:2943296 HeapReleased:0 TotalAlloc:767716501 Mallocs:30080694 Frees:30033334 GCSys:62753}
panic: runtime error: out of memory
Error: failed to run main module `transform.wasm`
Caused by:
0: failed to invoke command default
1: error while executing at wasm backtrace:
0: 0x109b3 - <unknown>!runtime.runtimePanicAt
1: 0x1952 - <unknown>!runtime.alloc
2: 0x202a - <unknown>!runtime.sliceAppend
3: 0x11764 - <unknown>!_start
note: using the `WASMTIME_BACKTRACE_DETAILS=1` environment variable may show more debugging information
2: wasm trap: wasm `unreachable` instruction executed
FWIW the leak reproduces on all platforms, not just wasm, if that makes debugging easier
I looked into this; it doesn't happen for me on native, which leads me to believe that it's just the a problem of a conservative garbage collector working with a 32-bit address space accidentally pinning large allocations. I'll leave this open for now incase I want to take another look at this.
I think I am encountering the same problem developing https://github.com/goui-org/goui, which runs with wasm (-gc=conservative) in the browser. After updating the dom millions of times, I get this:
panic: runtime error: out of memory
@dgryski What is the solution?
I added some logging in gc_* src, and it does free bytes regularly, but still must call growHeap many times.
I also logged runtime memory stats to confirm HeapInuse always increases and never decreases. Does this rule out fragmentation?
I already tried -gc=precise, and that causes nil ptr dereference panics and other panics.