FederatedAI
ChatGLM-6B TypeError: Object of type set is not JSON serializable
File "demo.py", line 100, in
pipeline.compile() │ └ <function PipeLine.compile at 0x7f0a0aa4ac10> └ <pipeline.backend.pipeline.PipeLine object at 0x7f0b293d0ca0> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/site-packages/pipeline/backend/pipeline.py", line 428, in compile self._train_conf = self._construct_train_conf() │ │ │ └ <function PipeLine._construct_train_conf at 0x7f0a0aa4a550> │ │ └ <pipeline.backend.pipeline.PipeLine object at 0x7f0b293d0ca0> │ └ {'dsl_version': 2, 'initiator': {'role': 'guest', 'party_id': 9999}, 'role': {'guest': [9999], 'host': [10000], 'arbiter': [9... └ <pipeline.backend.pipeline.PipeLine object at 0x7f0b293d0ca0> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/site-packages/pipeline/backend/pipeline.py", line 395, in _construct_train_conf LOGGER.debug(f"self._train_conf: \n {json.dumps(self._train_conf, indent=4, ensure_ascii=False)}") │ └ <function Logger.debug at 0x7f0a0dd301f0> └ <loguru.logger handlers=[(id=1, level=20, sink= ), (id=2, level=20, sink='/data/zhihao/anaconda3/envs/fate_env/lib/pyt... File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/init.py", line 234, in dumps return cls( └ <class 'json.encoder.JSONEncoder'> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 201, in encode chunks = list(chunks) └ <generator object _make_iterencode. ._iterencode at 0x7f0a083f4f20> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 431, in _iterencode yield from _iterencode_dict(o, _current_indent_level) │ │ └ 0 │ └ {'dsl_version': 2, 'initiator': {'role': 'guest', 'party_id': 9999}, 'role': {'guest': [9999], 'host': [10000], 'arbiter': [9... └ <function _make_iterencode. ._iterencode_dict at 0x7f0a0aa563a0> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 405, in _iterencode_dict yield from chunks └ <generator object _make_iterencode. ._iterencode_dict at 0x7f0a08388430> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 405, in _iterencode_dict yield from chunks └ <generator object _make_iterencode. ._iterencode_dict at 0x7f0a083883c0> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 405, in _iterencode_dict yield from chunks └ <generator object _make_iterencode. ._iterencode_dict at 0x7f0a083884a0> [Previous line repeated 7 more times] File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 438, in _iterencode o = _default(o) │ └ {'query_key_value'} └ <bound method JSONEncoder.default of <json.encoder.JSONEncoder object at 0x7f0a0aa58b80>> File "/data/zhihao/anaconda3/envs/fate_env/lib/python3.8/json/encoder.py", line 179, in default raise TypeError(f'Object of type {o.class.name} '
TypeError: Object of type set is not JSON serializable
解决了，我把源码给改了，能跑起来了，但不知道会不会造成问题。 pipeline.py的_construct_train_conf()函数里
if "common" in param_conf:
common_param_conf = param_conf["common"]
# 转换 role_param_conf 中的 set 为 list
check_for_sets_and_convert(common_param_conf)
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "common" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["common"] = {}
self._train_conf["component_parameters"]["common"].update(common_param_conf)
if "role" in param_conf:
role_param_conf = param_conf["role"]
# 转换 role_param_conf 中的 set 为 list
check_for_sets_and_convert(role_param_conf)
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "role" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["role"] = {}
self._train_conf["component_parameters"]["role"] = tools.merge_dict(
role_param_conf, self._train_conf["component_parameters"]["role"])
然后加了set转为list的函数
def convert_sets_to_lists(obj):
"""递归转换字典中的所有 set 为 list。"""
if isinstance(obj, set):
return list(obj)
elif isinstance(obj, dict):
for key in obj:
obj[key] = convert_sets_to_lists(obj[key])
elif isinstance(obj, list):
obj = [convert_sets_to_lists(item) for item in obj]
return obj
def check_for_sets_and_convert(dictionary):
"""检查字典中的 set 并转换为 list，用于确保 JSON 序列化的兼容性。"""
for key, value in dictionary.items():
dictionary[key] = convert_sets_to_lists(value)
是不是用了新版本的peft(>=0.6.0)？是的话，to_dict之间，执行一下
lora_config.target_modules = list(lora_config.target_modules)
