CacheLib
CacheLib copied to clipboard
facebook
Failed to assign buffers. All buffers(2) are being used
Describe the bug running kvcache and met fail to assign buffer issue, tried to incrase but not able to increase, it seems it is stuck at 2 in the following
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheNumInMemBuffers": "2",
Expected behavior Please provide some guidance on how to increase this "blockCacheNumInMemBuffers"
Related log
===JSON Config===
{
"cache_config": {
"cacheSizeMB": 43000,
"cacheDir": "/var/tmp/cachelib_meta",
"allocFactor": 1.08,
"maxAllocSize": 524288,
"minAllocSize": 64,
"nvmCachePaths": ["/dev/nvme7n1"],
"nvmCacheSizeMB": 952320,
"navyBlockDevice": true,
"navyEnableIoUring": false,
"navyQDepth": 32,
"deviceMaxWriteSize": 196608,
"navyReaderThreads": 32,
"navyWriterThreads": 16,
"navyBigHashBucketSize": 4096,
"navyBigHashSizePct": 10,
"navySmallItemMaxSize": 640,
"navyBlockSize": 4096,
"navyParcelMemoryMB": 6048,
"nvmAdmissionRetentionTimeThreshold": 7200,
"enableChainedItem": true,
"htBucketPower": 28,
"moveOnSlabRelease": false,
"poolRebalanceIntervalSec": 2,
"poolResizeIntervalSec": 2,
"rebalanceStrategy": "hits",
"writeAmpDeviceList": ["nvme7n1"]
},
"test_config": {
"opRatePerSec": 550000,
"opRateBurstSize": 200,
"generator": "replay",
"replayGeneratorConfig": { "ampFactor": 100 },
"repeatTraceReplay": true,
"repeatOpCount": true,
"onlySetIfMiss": false,
"numOps": 100000000000,
"numThreads": 10,
"prepopulateCache": true,
"enableLookaside": false,
"traceFileNames": [
"kvcache_traces_1.csv",
"kvcache_traces_2.csv",
"kvcache_traces_3.csv",
"kvcache_traces_4.csv",
"kvcache_traces_5.csv"
]
}
}
Welcome to OSS version of cachebench
I0828 11:59:32.596024 22913 KVReplayGenerator.h:106] Started KVReplayGenerator (amp factor 100, # of stressor threads 10)
I0828 11:59:32.596048 22914 ReplayGeneratorBase.h:218] [0] Opened trace file kvcache_traces_1.csv
I0828 11:59:32.596574 22914 ReplayGeneratorBase.h:179] New header detected: header "key,op,size,op_count,key_size" field map key -> 0, op -> 1, size -> 2, op_count -> 3, key_size -> 4
E0828 11:59:32.638914 22913 Cache.h:498] Exception fetching nand writes for nvme7n1. Msg: Vendor not recogized in device model number solidigm snbph27x076tgf
I0828 11:59:32.639419 22913 Cache.h:622] Configuring NVM cache: simple file /dev/nvme7n1 size 952320 MB
I0828 11:59:32.640034 22913 Cache.h:711] Using the following nvm config{
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::QDepth": "32",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionPolicy": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbBaseSize": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbFactorLowerBound": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbFactorUpperBound": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbability": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionSuffixLen": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionWriteRate": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashBucketBfSize": "8",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashBucketSize": "4096",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashSizePct": "10",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashSmallItemMaxSize": "640",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheCleanRegionThreads": "1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheCleanRegions": "1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheDataChecksum": "true",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheLru": "true",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheNumInMemBuffers": "2",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheRegionSize": "16777216",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheReinsertionHitsThreshold": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheReinsertionPctThreshold": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheSegmentedFifoSegmentRatio": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockSize": "4096",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::deviceMaxWriteSize": "196608",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::deviceMetadataSize": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::enableFDP": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::fileName": "/dev/nvme7n1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::fileSize": "998579896320",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::ioEngine": "libaio",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxConcurrentInserts": "1000000",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxNumReads": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxNumWrites": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxParcelMemoryMB": "6048",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxWriteRate": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::navyReqOrderingShards": "21",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::raidPaths": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::readerThreads": "32",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::stackSize": "16384",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::truncateFile": "false",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::writerThreads": "16"
I0828 11:59:32.640034 22913 Cache.h:711] }
I0828 11:59:32.641212 22913 Cache.h:750] Failed to attach for reason: Unable to find any segment with name shm_info
E0828 11:59:37.637370 22913 NvmCacheState.cpp:135] unable to deserialize nvm metadata file: no content in file: /var/tmp/cachelib_meta/NvmCacheState
I0828 11:59:37.647208 22913 Device.cpp:1085] Cache file: /dev/nvme7n1 size: 998579896320 truncate: 0
I0828 11:59:37.647334 22913 Device.cpp:970] Created device with num_devices 1 size 998579896320 block_size 4096,stripe_size 0 max_write_size 196608 max_io_size 196608 io_engine libaio qdepth 32,num_fdp_devices 0
I0828 11:59:37.813466 22913 NavySetup.cpp:243] metadataSize: 4992897024
I0828 11:59:37.813526 22913 NavySetup.cpp:245] Setting up engine pair 0
I0828 11:59:37.813547 22913 NavySetup.cpp:111] bighashStartingLimit: 4992897024 bigHashCacheOffset: 898721906688 bigHashCacheSize: 99857989632
I0828 11:59:37.813560 22913 NavySetup.cpp:259] blockCacheSize 893729009664
I0828 11:59:37.813572 22913 NavySetup.cpp:156] blockcache: starting offset: 4992897024, block cache size: 893722296320
I0828 11:59:37.813708 22913 LruPolicy.cpp:35] LRU policy: expected 53270 regions
I0828 11:59:37.960658 22913 BigHash.cpp:93] BigHash created: buckets: 24379392, bucket size: 4096, base offset: 898721906688
I0828 11:59:37.960726 22913 BigHash.cpp:102] Reset BigHash
I0828 11:59:37.985409 22913 RegionManager.cpp:50] 53270 regions, 16777216 bytes each
I0828 11:59:38.013070 22964 RegionManager.cpp:68] region_manager_0 started
I0828 11:59:38.064168 22913 Allocator.cpp:39] Enable priority-based allocation for Allocator. Number of priorities: 1
I0828 11:59:38.064239 22913 BlockCache.cpp:145] Block cache created
I0828 11:59:38.064573 22913 Driver.cpp:70] Max concurrent inserts: 1000000
I0828 11:59:38.064597 22913 Driver.cpp:71] Max parcel memory: 6341787648
I0828 11:59:38.064612 22913 Driver.cpp:72] Use Write Estimated Size: false
I0828 11:59:38.064634 22913 Driver.cpp:209] Reset Navy
I0828 11:59:38.064662 22913 BigHash.cpp:102] Reset BigHash
I0828 11:59:38.080072 22913 BlockCache.cpp:707] Reset block cache
Total 1000000.00M ops to be run
11:59:38 0.00M ops completed. Hit Ratio 90.00% (RAM 91.00%, NVM 0.00%)
12:00:38 32.31M ops completed. Hit Ratio 9.86% (RAM 9.86%, NVM 0.00%)
12:01:38 64.62M ops completed. Hit Ratio 20.65% (RAM 20.65%, NVM -0.00%)
12:02:38 96.86M ops completed. Hit Ratio 33.11% (RAM 33.11%, NVM 0.00%)
12:03:38 129.39M ops completed. Hit Ratio 34.99% (RAM 34.99%, NVM -0.00%)
12:04:38 161.69M ops completed. Hit Ratio 29.53% (RAM 29.53%, NVM 0.00%)
12:05:38 194.06M ops completed. Hit Ratio 35.63% (RAM 35.63%, NVM -0.00%)
12:06:38 226.39M ops completed. Hit Ratio 37.71% (RAM 37.71%, NVM 0.00%)
12:07:38 258.86M ops completed. Hit Ratio 39.27% (RAM 39.27%, NVM 0.00%)
12:08:38 291.29M ops completed. Hit Ratio 33.95% (RAM 33.95%, NVM 0.00%)
12:09:38 323.45M ops completed. Hit Ratio 39.75% (RAM 39.75%, NVM -0.00%)
12:10:39 355.93M ops completed. Hit Ratio 43.17% (RAM 43.17%, NVM 0.00%)
12:11:39 388.36M ops completed. Hit Ratio 37.14% (RAM 37.14%, NVM 0.00%)
12:12:39 420.80M ops completed. Hit Ratio 44.84% (RAM 44.84%, NVM -0.00%)
12:13:39 453.37M ops completed. Hit Ratio 44.40% (RAM 44.40%, NVM 0.00%)
12:14:39 485.79M ops completed. Hit Ratio 44.13% (RAM 44.13%, NVM 0.00%)
I0828 12:15:10.938798 22964 Device.cpp:780] [ctx_0] Created new async io context with qdepth 32 io_engine libaio
E0828 12:15:12.335407 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:15:25.528841 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:15:36.335482 22953 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:15:39 517.90M ops completed. Hit Ratio 42.02% (RAM 42.01%, NVM 0.01%)
E0828 12:15:53.231555 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:10.277482 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:20.276806 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:30.998905 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:16:39 549.37M ops completed. Hit Ratio 45.38% (RAM 45.34%, NVM 0.07%)
E0828 12:16:42.982179 22959 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:58.429018 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:08.687214 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:21.591422 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:36.023783 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:17:39 580.88M ops completed. Hit Ratio 48.28% (RAM 48.21%, NVM 0.14%)
E0828 12:17:47.105793 22962 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:01.100442 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:11.100159 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:31.403505 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:18:39 612.28M ops completed. Hit Ratio 36.50% (RAM 36.39%, NVM 0.17%)
E0828 12:18:41.835194 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:55.052945 22957 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:13.595917 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:25.874168 22949 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:37.042677 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:19:39 643.29M ops completed. Hit Ratio 45.51% (RAM 45.38%, NVM 0.24%)
E0828 12:19:47.044314 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:57.098433 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:13.160506 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:28.029697 22951 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:38.397520 22955 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:20:39 674.23M ops completed. Hit Ratio 45.55% (RAM 45.33%, NVM 0.40%)
E0828 12:20:49.040838 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:01.804480 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:12.660606 22956 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:31.857285 22949 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:21:39 704.84M ops completed. Hit Ratio 45.94% (RAM 45.71%, NVM 0.42%)
E0828 12:21:43.260118 22958 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:58.208551 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:08.663174 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:24.253245 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:22:40 735.80M ops completed. Hit Ratio 51.25% (RAM 50.99%, NVM 0.53%)
E0828 12:22:44.863645 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:56.354596 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:07.300051 22951 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:18.585624 22955 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:37.090524 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:23:40 766.29M ops completed. Hit Ratio 44.19% (RAM 43.90%, NVM 0.52%)
E0828 12:23:49.779486 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:24:03.425178 22953 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:24:19.538671 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
