manticoresoftware
goroutine error on query
Hi,
Hope you are all well !
I cannot insert this query, and I have an error
REPLACE into limostar (
id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license
) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56 0100 CET','2014-07-31 23:56:31 0200 CEST','2020-02-18 11:32:08 0100 CET','github','unkown')
"\x02"
manticore.apibuf{}
"\x00"
panic: runtime error: slice bounds out of range [2:0]
goroutine 1 [running]:
github.com/x0rzkov/go-sdk/manticore.(*apibuf).getLsbWord(0xc000733440, 0x10002)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/sphinxql.go:330 0x1af
github.com/x0rzkov/go-sdk/manticore.(*Sqlresult).parseOK(0xc0007334d8, 0xc000733440)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/sphinxql.go:270 0x84
github.com/x0rzkov/go-sdk/manticore.(*Sqlresult).parseChain(0xc0007334d8, 0xc008777e40, 0x18)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/sphinxql.go:58 0x200
github.com/x0rzkov/go-sdk/manticore.parseSphinxqlAnswer.func1(0xc008777e40, 0xc008770100, 0xc005fb5b88)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/sphinxql.go:22 0x8c
github.com/x0rzkov/go-sdk/manticore.(*Client).netQuery(0xc000280880, 0x8, 0xc008777e20, 0x4feb030, 0x4f30860, 0xc0004781c0, 0x1, 0x1)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/client.go:225 0x1ca
github.com/x0rzkov/go-sdk/manticore.(*Client).Sphinxql(0xc000280880, 0xc005fb4f00, 0x26e, 0x1, 0x13, 0xc005fb4f00, 0x26e, 0x0)
/Users/x0rzkov/go/pkg/mod/github.com/x0rzkov/[email protected]/manticore/manticore.go:398 0x70
github.com/lucmski/limo/cmd.sphinxStars(0x5150500, 0xc0000ac000, 0xc0003ccb40, 0x1, 0x1)
/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/sphinxql.go:161 0xb1a
github.com/lucmski/limo/cmd.glob..func14(0x5a1b760, 0xc0003ccb40, 0x1, 0x1)
/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/sphinxql.go:48 0xca
github.com/spf13/cobra.(*Command).execute(0x5a1b760, 0xc0003ccaf0, 0x1, 0x1, 0x5a1b760, 0xc0003ccaf0)
/Users/x0rzkov/go/pkg/mod/github.com/spf13/[email protected]/command.go:833 0x2aa
github.com/spf13/cobra.(*Command).ExecuteC(0x5a1afe0, 0x403d4ba, 0x599d8a0, 0xc000000180)
/Users/x0rzkov/go/pkg/mod/github.com/spf13/[email protected]/command.go:917 0x2fb
github.com/spf13/cobra.(*Command).Execute(...)
/Users/x0rzkov/go/pkg/mod/github.com/spf13/[email protected]/command.go:867
github.com/lucmski/limo/cmd.Execute()
/Users/x0rzkov/go/src/github.com/lucmski/limo/cmd/root.go:52 0x31
main.main()
/Users/x0rzkov/go/src/github.com/lucmski/limo/main.go:8 0x20
Manticore Config
#!/bin/sh
ip=`hostname -i`
cat << EOF
index limostar {
type = rt
rt_mem_limit = 1024M
path = /var/lib/manticore/data/limostar
rt_attr_bigint = remote_id
rt_field = avatar
rt_attr_string = owner
rt_attr_string = name
rt_attr_string = url
rt_attr_string = language
rt_attr_bigint = watchers
rt_attr_bigint = stars
rt_attr_bigint = forks
rt_attr_bigint = size
rt_attr_string = license
rt_attr_timestamp = pushed
rt_attr_timestamp = created
rt_attr_timestamp = starred
rt_attr_string = service_id
rt_attr_multi = tags
rt_attr_multi = topics
rt_field = fullname
rt_field = description
rt_field = service_id
rt_field = homepage
rt_field = url
# stored_fields = fullname,description,service_id
min_infix_len = 2
}
indexer
{
mem_limit = 64M
}
index limotopic {
type = rt
rt_mem_limit = 1024M
path = /var/lib/manticore/data/limotopic
rt_attr_bigint = remote_id
rt_attr_string = service_id
rt_attr_string = url
rt_attr_string = tag
rt_field = url
rt_field = tag
rt_field = service_id
# stored_fields = tag,url,service_id
min_infix_len = 2
}
index limo
{
type = distributed
local = limostar
local = limotopic
}
index pq {
type = percolate
path = /var/lib/manticore/data/pq
min_infix_len = 4
}
searchd {
listen = 9306:mysql41
listen = /var/run/mysqld/mysqld.sock:mysql41
listen = $ip:9312
listen = 9308:http
# more info about replication you can find at
# https://docs.manticoresearch.com/latest/html/replication.html
listen = $ip:9315-9325:replication
log = /var/log/manticore/searchd.log
# you can also send query_log to /dev/stdout to be shown in docker logs
query_log = /var/log/manticore/query.log
read_timeout = 15
max_children = 30
pid_file = /var/run/manticore/searchd.pid
seamless_rotate = 1
preopen_indexes = 1
unlink_old = 1
# workers = thread_pool
max_matches = 2000
workers = threads
subtree_docs_cache = 8M
subtree_hits_cache = 16M
binlog_path = /var/lib/manticore/data
max_packet_size = 128M
mysql_version_string = 5.5.21
data_dir = /var/lib/manticore/replication
}
EOF
Any reason why ?
Cheers,
X.
@x0rzkov Hello. Can you please provide the script/code snippet which we can use to reproduce this issue?
Hi,
Hope you are all well and thanks for the reply !
I just made you a repository in order to test my use case. Ref. https://github.com/x0rzkov/manticore-test
How to run:
Pre-requisites
- docker
- docker-compose
Before
tar xvf manticore-dump.sql.tar.gz
Run
docker-compose up --build
Importer (options)
% importer -h
-h, --help display help
-f, --input-file string input-file (default "/opt/manticore/data/manticore-dump.sql")
-m, --manticore-host string input-file (default "localhost")
-p, --manticore-port uint16 input-file (default 9312)
-j, --parallel-jobs int parallel-jobs (default 2)
Quick important note: I do not understand how to add the rt_attr_multi tags in the limostar index. How do you build the query for real time indexes ?
Cheers, X
Hi @x0rzkov
I get the following error and backtrace:
manticore-importer | time="2020-02-24T06:36:02Z" level=info msg="query:REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('253b66546bf9e540ac02035f014e62e3', '197773483','github','https://github.com/AbduallahAtta/Social-Note.git','room')"
manticore-importer | panic: runtime error: index out of range [1] with length 1
manticore-importer |
manticore-importer | goroutine 7882 [running]:
manticore-importer | encoding/binary.littleEndian.Uint16(...)
manticore-importer | /usr/local/go/src/encoding/binary/binary.go:52
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*apibuf).getLsbWord(...)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/sphinxql.go:325
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Sqlresult).parseOK(0xc0000c7d50, 0xc0000c7cb8)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/sphinxql.go:267 +0x166
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Sqlresult).parseChain(0xc0000c7d50, 0xc00031b460, 0x18)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/sphinxql.go:56 +0x27c
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.parseSphinxqlAnswer.func1(0xc00031b460, 0xc000310100, 0xc0002dd080)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/sphinxql.go:20 +0x8c
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Client).netQuery(0xc0000c0000, 0x8, 0xc00031b440, 0x57c6d0, 0x56dae0, 0xc0004c2000, 0x2, 0x2)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/client.go:225 +0x1ca
manticore-importer | github.com/manticoresoftware/go-sdk/manticore.(*Client).Sphinxql(0xc0000c0000, 0xc00032b080, 0xb4, 0x2, 0x2, 0xb4, 0x0, 0x0)
manticore-importer | /go/pkg/mod/github.com/manticoresoftware/[email protected]/manticore/manticore.go:398 +0x70
manticore-importer | main.main.func1(0xc0000b2140, 0xc0000c0280, 0xc0000c0000, 0x0, 0x0)
manticore-importer | /go/src/github.com/x0rzkov/manticore-test/main.go:64 +0x17a
manticore-importer | created by main.main
manticore-importer | /go/src/github.com/x0rzkov/manticore-test/main.go:58 +0x607
manticore-importer exited with code 2
Since it's different from the mentioned in this issue's body I want to ask - is the the error which concerns you now?
I do not understand how to add the rt_attr_multi tags in the limostar index.
It's already there:
snikolaev@dev:~/manticore-test$ cat manticore.conf |grep -A 20 "index limostar"
index limostar {
type = rt
rt_mem_limit = 1024M
path = /var/lib/manticore/data/limostar
rt_attr_bigint = remote_id
rt_field = avatar
rt_attr_string = owner
rt_attr_string = name
rt_attr_string = url
rt_attr_string = language
rt_attr_bigint = watchers
rt_attr_bigint = stars
rt_attr_bigint = forks
rt_attr_bigint = size
rt_attr_string = license
rt_attr_timestamp = pushed
rt_attr_timestamp = created
rt_attr_timestamp = starred
rt_attr_string = service_id
rt_attr_multi = tags
How do you build the query for real time indexes ?
What query do you mean?
Now I have another error,
manticore-importer | time="2020-02-24T07:22:08Z" level=info msg="query:REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('5d10614167fa5e88a26b78d1e16283cd', '43965481','github','https://github.com/0010-IO/gealts.git','elixir')"
manticore-importer | time="2020-02-24T07:22:08Z" level=info msg="query error: REPLACE into limotopic (id,remote_id,service_id,url,tag) VALUES ('5d10614167fa5e88a26b78d1e16283cd', '43965481','github','https://github.com/0010-IO/gealts.git','elixir')"
manticore-importer | time="2020-02-24T07:22:08Z" level=fatal msg=EOF
manticore-importer exited with code 1
So the idea is to check if manticore's go-sdk can absorb lots of data. That's simply it, and this repo is a good occasion to test your sdk.
For How do you build the query for real time indexes ?
What is the sphinxql query to add multiattr to limostar index ? Give me just an example how to add all the tags per star.
Thanks in advance for your insights per advance.
Ps. It is quite urgent as we have to decide if we have to continue to use manticore or not.
Try workers=thread_pool. In this case I get the error as I showed above - "index out of range [1] with length 1" and it seems it's a bug in the SDK.
What is the sphinxql query to add multiattr to limostar index ?
mysql> alter table limostar add column tags2 multi;
Query OK, 0 rows affected (0.07 sec)
mysql> alter table limostar add column tags3 multi64;
Query OK, 0 rows affected (0.04 sec)
mysql> alter table limostar add column tags4 string;
Query OK, 0 rows affected (0.05 sec)
Give me just an example how to add all the tags per star.
mysql> insert into limostar(tags2,tags3,tags4) values((1,2,3),(1,2,3),'a b c');
Query OK, 1 row affected (0.00 sec)
mysql> select tags4 from limostar where tags4 any ('c', 'd');
+-------+
| tags4 |
+-------+
| a b c |
+-------+
1 row in set (0.00 sec)
Hi,
Thanks a lot for the reply, I am working with @x0rzkov and will try out now the query for tags.
When do you think that the sdk can be fixed ?
Best Regards, Luc Michalski
@lucmichalski
When do you think that the sdk can be fixed ?
I'm afraid not today. Perhaps not even this week. What I can recommend is you could just use a mysql Go library to connect to the mysql's Manticore port (9306 in your docker compose). The binary protocol which the SDK implements might give very insignificant performance benefit, what it gives is mostly just an abstraction layer for more convenient development, but it seems that in your app (at least what you have shared) it may be not that important.
Also if you're looking for max INSERT/REPLACE throughput and you have lots of incoming docs make sure you do batching as it increases the throughput significantly. Here is what we recently measured on one of our customers's single server with standard linux mysql client + GNU parallel:
With batch=10K+32 parallel workers I could get the insert throughput of a single RT index =
14892 docs per second
13.71MB (of source data + sql overhead) per second
= 1.12TB per day
There’re 32 cores on the server. They were ~1/3 loaded:
[root@sehs01 ms_vs_es]# dstat 30
You did not select any stats, using -cdngy by default.
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read writ| recv send| in out | int csw
2 1 97 0 0 0|1165k 1000k| 0 0 | 0 0 | 12k 3815
35 2 62 0 0 0| 0 9990k| 20k 24k| 0 0 | 74k 6414
31 1 68 0 0 0| 0 25M| 22k 62k| 0 0 | 81k 4348
31 3 66 0 0 0| 0 8491k| 23k 73k| 0 0 | 86k 6229
27 1 72 0 0 0| 0 7496k| 41k 101k| 0 0 | 43k 4175
28 2 70 0 0 0| 26k 6686k| 21k 27k| 0 0 | 78k 5874
25 1 74 0 0 0| 0 6049k| 23k 29k| 0 0 | 36k 3749
27 2 71 0 0 0|1638B 6129k| 20k 18k| 0 0 | 37k 5690
25 1 74 0 0 0| 0 5736k| 22k 63k| 0 0 | 30k 3641
27 2 71 0 0 0| 0 11M| 21k 107k| 0 0 | 40k 5527
25 1 74 0 0 0| 273B 12M| 24k 65k| 0 0 | 30k 3661
26 2 72 0 0 0| 546B 10M| 16k 15k| 0 0 | 35k 5478
24 1 75 0 0 0| 956B 11M| 20k 31k| 0 0 | 23k 3553
26 3 71 0 0 0| 137B 5713k| 15k 70k| 0 0 | 48k 5755
24 1 75 0 0 0| 13k 16M| 18k 25k| 0 0 | 54k 4340
37 3 61 0 0 0|1092B 28M| 16k 37k| 0 0 | 105k 7058
30 1 69 0 0 0| 273B 8930k| 18k 105k| 0 0 | 64k 4160
30 2 68 0 0 0| 273B 8018k| 31k 76k| 0 0 | 48k 6180
28 1 71 0 0 0|3277B 8266k| 27k 26k| 0 0 | 73k 4239
27 2 71 0 0 0| 273B 7182k| 18k 16k| 0 0 | 38k 5732
25 1 74 0 0 0| 0 7102k| 18k 80k| 0 0 | 27k 3718
25 2 73 0 0 0| 273B 6775k| 21k 45k| 0 0 | 37k 5721
10 1 89 0 0 0| 137B 2957k| 23k 23k| 0 0 | 44k 2817
You can additional enable sharding:
10K batch, 32 workers, 8 shards, NO OPTIMIZE:
[root@sehs01 ms_vs_es]# time parallel -j32 --pipepart -a all.10000.sharded.sql "mysql -P8312 -h0"
real 6m22.163s
user 7m6.135s
sys 0m52.999s
or 23951437 / (6*60+22) = 62700 docs per second or 57,75MB per second or 4,75TB per day
The load was:
[root@sehs01 ms_vs_es]# dstat 30
You did not select any stats, using -cdngy by default.
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read writ| recv send| in out | int csw
2 1 97 0 0 0|1119k 1079k| 0 0 | 0 0 | 12k 3837
89 5 6 0 0 0|3004B 47M| 14k 50k| 0 0 | 372k 12k
88 3 9 0 0 0|2185B 45M| 11k 12k| 0 0 | 243k 11k
85 4 11 0 0 0|7646B 46M| 12k 13k| 0 0 | 113k 12k
84 2 13 0 0 0|1911B 47M| 15k 43k| 0 0 | 96k 10k
82 4 13 0 0 0| 245k 46M| 11k 17k| 0 0 | 141k 12k
52 2 46 0 0 0| 193k 32M| 12k 33k| 0 0 | 100k 7965
5 2 93 0 0 0| 410B 11M| 13k 55k| 0 0 | 28k 4542
4 1 96 0 0 0| 135k 14M| 15k 36k| 0 0 |6699 2771
5 2 94 0 0 0|2867B 8923k| 11k 33k| 0 0 |8586 4511
3 0 96 0 0 0| 241k 13M| 15k 18k| 0 0 |4870 2382
5 2 93 0 0 0| 410B 13M| 13k 28k| 0 0 | 17k 4487
3 1 96 0 0 0| 878B 16M| 19k 51k| 0 0 |6771 2875 ^C
Caution: it doesn't include time to OPTIMIZE.
It works fine with the mysql driver, even with 15 parallel jobs !
Please, update us when you have fixed the sdk.
Thanks in advance
Hello. I've looked into you code and see following pattern: you've created one client, and then try to use it in parallel from bunch of goroutines. That way it will not work anyway: once client connected, you can use it serial way, sending query-by-query and reading response after each of them. There is only one socket opened, and daemon expects one flow of commands; you can't share one and same among parallel works.
So, you just need to create an instance of client for every separate coro. Then each one will have own connection which is not shared with anothers, and the crashes must go out.
I tried with only one thread, and the bug is also triggered, at list with the go-sdk.
I've just tried minute ago:
cl := NewClient()
res, err := cl.Sphinxql(`REPLACE into limostar (id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56 +0100 CET','2014-07-31 23:56:31 +0200 CEST','2020-02-18 11:32:08 +0100 CET','github','unkown')`)
fmt.Println(res, err)
and get [Query OK, 1 rows affected] <nil>.
Full test clause looking like this:
func TestClient_Sphinxql_insert(t *testing.T) {
cl := NewClient()
_, err := cl.netQuery(commandPersist, buildBoolRequest(true), nil)
res, err := cl.Sphinxql(`REPLACE into limostar (id,remote_id,avatar,owner,name,fullname,description,homepage,url,language,watchers,stars,forks,size,pushed,created,starred,service_id,license) VALUES ('96285','22486533','https://avatars2.githubusercontent.com/u/2347603?v=4','00-Evan','shattered-pixel-dungeon','00-Evan/shattered-pixel-dungeon','Traditional roguelike game with pixel-art graphics and simple interface','unkown','https://github.com/00-Evan/shattered-pixel-dungeon.git','Java','623','623','279','29628','2020-01-13 02:41:56 +0100 CET','2014-07-31 23:56:31 +0200 CEST','2020-02-18 11:32:08 +0100 CET','github','unkown')`)
fmt.Println(res, err)
res, err = cl.Sphinxql(`replace into testrt values(2,'another subject', 'more content', 15)`)
fmt.Println(res, err)
res, err = cl.Sphinxql(`replace into testrt values(5,'again subject', 'one more content', 10)`)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Println(res)
}
}
it works as expected. What is daemon version you run on? May be something wrong there?