microsoft
Segmentation fault (core dumped)
Describe the bug
Test SPTag wrapper on python 2.7, ubuntu 18.04 TRY x is FLOAT data. Got error Segmentation fault (core dumped)
import SPTAG
import numpy as np
n = 100
k = 3
r = 3
def testBuild(algo, distmethod, x, out):
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i.SetBuildParam("NumberOfThreads", '4')
i.SetBuildParam("DistCalcMethod", distmethod)
ret = i.Build(x.tobytes(), x.shape[0])
i.Save(out)
def testSearch(index, q, k):
j = SPTAG.AnnIndex.Load(index)
for t in range(q.shape[0]):
result = j.Search(q[t].tobytes(), k)
print(result[0]) # ids
print(result[1]) # distances
def Test(algo, distmethod):
x = np.ones((n, 10), dtype=np.float32) * np.reshape(np.random.uniform(low=0.5, high=13.3, size=(n,)), (n,1))
print('x.shape: ', str(x.shape))
print(x[1,])
print("Build.............................")
testBuild(algo, distmethod, x, 'testindices')
testSearch('testindices', q, k)
if __name__ == '__main__':
Test('BKT', 'L2')
# Test('KDT', 'L2')
Log console:
('x.shape: ', '(100, 10)') [ 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413 0.60697413] Build............................. Setting NumberOfThreads with value 4 Setting DistCalcMethod with value L2 Save Data To testindices/vectors.bin Save Data (0, 1) Finish! Save BKT to testindices/tree.bin Segmentation fault (core dumped)
Currently, we only support np.float32 type. np.random.uniform will generate np.float64 data. Try to use np.random.uniform(low=0.5, high=13.3, size=(n,)).astype(np.float32)
@MaggieQi I also got segmentation fault with np.float32 type. python 2.7, ubuntu 14.04
import SPTAG
import numpy as np
n = 100
k = 3
r = 3
def testBuild(algo, distmethod, x, out):
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i.SetBuildParam("NumberOfThreads", '4')
i.SetBuildParam("DistCalcMethod", distmethod)
ret = i.Build(x.tobytes(), x.shape[0])
i.Save(out)
def testBuildWithMetaData(algo, distmethod, x, s, out):
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i.SetBuildParam("NumberOfThreads", '4')
i.SetBuildParam("DistCalcMethod", distmethod)
if i.BuildWithMetaData(x.tobytes(), s, x.shape[0]):
i.Save(out)
def testSearch(index, q, k):
j = SPTAG.AnnIndex.Load(index)
for t in range(q.shape[0]):
result = j.Search(q[t].tobytes(), k)
print (result[0]) # ids
print (result[1]) # distances
def testSearchWithMetaData(index, q, k):
j = SPTAG.AnnIndex.Load(index)
j.SetSearchParam("MaxCheck", '1024')
for t in range(q.shape[0]):
result = j.SearchWithMetaData(q[t].tobytes(), k)
print (result[0]) # ids
print (result[1]) # distances
print (result[2]) # metadata
def testAdd(index, x, out, algo, distmethod):
if index != None:
i = SPTAG.AnnIndex.Load(index)
else:
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i.SetBuildParam("NumberOfThreads", '4')
i.SetBuildParam("DistCalcMethod", distmethod)
if i.Add(x.tobytes(), x.shape[0]):
i.Save(out)
def testAddWithMetaData(index, x, s, out, algo, distmethod):
if index != None:
i = SPTAG.AnnIndex.Load(index)
else:
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i = SPTAG.AnnIndex(algo, 'Float', x.shape[1])
i.SetBuildParam("NumberOfThreads", '4')
i.SetBuildParam("DistCalcMethod", distmethod)
if i.AddWithMetaData(x.tobytes(), s, x.shape[0]):
i.Save(out)
def testDelete(index, x, out):
i = SPTAG.AnnIndex.Load(index)
ret = i.Delete(x.tobytes(), x.shape[0])
print (ret)
i.Save(out)
def Test(algo, distmethod):
x = np.ones((n, 10), dtype=np.float32) * np.reshape(np.arange(n, dtype=np.float32), (n, 1))
q = np.ones((r, 10), dtype=np.float32) * np.reshape(np.arange(r, dtype=np.float32), (r, 1)) * 2
print x
print q
m = ''
for i in range(n):
m += str(i) + '\n'
print ("Build.............................")
testBuild(algo, distmethod, x, 'testindices')
testSearch('testindices', q, k)
print ("Add.............................")
testAdd('testindices', x, 'testindices', algo, distmethod)
testSearch('testindices', q, k)
print ("Delete.............................")
testDelete('testindices', q, 'testindices')
testSearch('testindices', q, k)
print ("AddWithMetaData.............................")
testAddWithMetaData(None, x, m, 'testindices', algo, distmethod)
print ("Delete.............................")
testSearchWithMetaData('testindices', q, k)
testDelete('testindices', q, 'testindices')
testSearchWithMetaData('testindices', q, k)
if __name__ == '__main__':
Test('BKT', 'L2')
Test('KDT', 'L2')
Build............................. Setting NumberOfThreads with value 4 Setting DistCalcMethod with value L2 Start to build BKTree 1 1 BKTree built, 101 100 build RNG graph! Refine 1 0%Refine RNG, graph acc:1 Refine 2 0%Refine RNG, graph acc:1 Build RNG Graph end! [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] Add............................. Setting NumberOfThreads with value 4 Setting DistCalcMethod with value L2 [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] [-1, -1, -1] [3.4028234663852886e+38, 3.4028234663852886e+38, 3.4028234663852886e+38] Delete............................. False Segmentation fault (core dumped)
Currently, we only support np.float32 type. np.random.uniform will generate np.float64 data. Try to use np.random.uniform(low=0.5, high=13.3, size=(n,)).astype(np.float32)
Passed, thanks
Trying to run IndexBuilder from command line like this: ./indexbuilder -d 1 -v Float -i ../all_question_embeddings.txt -o index/ -a BKT -t 4 but end up getting segmentation fault. I am using np.float32 as the data type.
Setting NumberOfThreads with value 4 Begin Subtask: 0, start offset position:0 Begin Subtask: 1, start offset position:2420000 Begin Subtask: 2, start offset position:4840000 Begin Subtask: 3, start offset position:7260000 Start to build BKTree 1 1 BKTree built, 4001 4000 build RNG graph! Parallel TpTree Partition begin Segmentation fault
