25_Soldier_Patrol_Soldier_Patrol_25_457.jpg
Traceback (most recent call last):
File "./train_net.py", line 134, in
restore=bool(int(args.restore)))
File "./lib/tiny/train.py", line 338, in train_net
sw.train_model(sess, epochs, restore=restore)
File "./lib/tiny/train.py", line 180, in train_model
score_cls_np, score_reg_np, prob_cls_np = sess.run(fetches=fetch_list, feed_dict=feed_dict)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 900, in run
run_metadata_ptr)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1316, in _do_run
run_metadata)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: ConcatOp : Ranks of all input tensors should match: shape[0] = [58,1] vs. shape[1] = [668697]
[[Node: gradients/concat = ConcatV2[N=4, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/device:GPU:0"](gradients/cond/GatherV2/Switch_grad/cond_grad, gradients/cond_1/GatherV2/Switch_grad/cond_grad, gradients/cond/GatherV2_1/Switch_grad/cond_grad, gradients/cond_1/GatherV2_1/Switch_grad/cond_grad, GatherV2_2/axis)]]
[[Node: Momentum/update/_1308 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_8094_Momentum/update", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Caused by op u'gradients/concat', defined at:
File "./train_net.py", line 134, in
restore=bool(int(args.restore)))
File "./lib/tiny/train.py", line 338, in train_net
sw.train_model(sess, epochs, restore=restore)
File "./lib/tiny/train.py", line 108, in train_model
train_op1 = opt.minimize(loss, var_list=[var for var in tvars], global_step=global_step)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 414, in minimize
grad_loss=grad_loss)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/training/optimizer.py", line 526, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 494, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 587, in _GradientsHelper
aggregation_method)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/ops/gradients_impl.py", line 953, in _AggregatedGrads
array_ops.concat([x.values for x in out_grad], 0),
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.py", line 1189, in concat
return gen_array_ops.concat_v2(values=values, axis=axis, name=name)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 953, in concat_v2
"ConcatV2", values=values, axis=axis, name=name)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 3392, in create_op
op_def=op_def)
File "/opt/dev/toolchain/Anaconda3/envs/py27/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): ConcatOp : Ranks of all input tensors should match: shape[0] = [58,1] vs. shape[1] = [668697]
[[Node: gradients/concat = ConcatV2[N=4, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/device:GPU:0"](gradients/cond/GatherV2/Switch_grad/cond_grad, gradients/cond_1/GatherV2/Switch_grad/cond_grad, gradients/cond/GatherV2_1/Switch_grad/cond_grad, gradients/cond_1/GatherV2_1/Switch_grad/cond_grad, GatherV2_2/axis)]]
[[Node: Momentum/update/_1308 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_8094_Momentum/update", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Any suggestions？
Hi,
I have tried a couple of possible enviroment simulations, but did not get the same error as yours, thus hard to pinpoint the source of error.
My guess is that the issue came from image IO, which caused the input tensor dimension mismatch.
Perhaps double checking the WIDER face dataset files integrity may help.
Thank you for your advice. @Timforce
Timforce
