FATE icon indicating copy to clipboard operation
FATE copied to clipboard

官方示例pipeline训练IMDB.CSV是否可以用api进行上传?

Open icutool opened this issue 1 year ago • 0 comments

这个例子我跑通了https://github.com/FederatedAI/FATE/blob/v1.11.3/doc/tutorial/pipeline/nn_tutorial/Bert-example.ipynb

然后我在fateboard下载了这个任务的dsl和conf两个文件,尝试使用flow data upload 和flow data submit进行提交任务训练 发现报错

ValueError: Invalid file path or buffer object type: <class 'fate_arch.computing.standalone._table.Table'>

但是似乎实现逻辑都是fate_client 所以我尝试修改配置文件也还是报错 无奈来求助了 感谢

uoload.json

{
    "file": "/opt/IMDB.csv",
    "head": 1,
    "partition": 10,
    "work_mode": 1,
    "table_name": "imdb",
    "namespace": "experiment"
}

dsl.json

{
	"components":{
		"nn_0":{
			"output":{
				"data":[
					"data"
				],
				"model":[
					"model"
				]
			},
			"input":{
				"data":{
					"validate_data":[
						"reader_1.data"
					],
					"train_data":[
						"reader_0.data"
					]
				}
			},
			"provider":"fate",
			"module":"HomoNN"
		},
		"reader_0":{
			"output":{
				"data":[
					"data"
				]
			},
			"provider":"fate_flow",
			"module":"Reader"
		},
		"reader_1":{
			"output":{
				"data":[
					"data"
				]
			},
			"provider":"fate_flow",
			"module":"Reader"
		},
		"eval_0":{
			"output":{
				"data":[
					"data"
				]
			},
			"input":{
				"data":{
					"data":[
						"nn_0.data"
					]
				}
			},
			"provider":"fate",
			"module":"Evaluation"
		}
	}
}

job_config.json

{
	"component_parameters":{
		"role":{
			"host":{
				"0":{
					"reader_0":{
						"table":{
							"name":"imdb",
							"namespace":"experiment"
						}
					},
					"reader_1":{
						"table":{
							"name":"imdb",
							"namespace":"experiment"
						}
					}
				}
			}
		},
		"common":{
			"nn_0":{
				"loss":{
					"reduce":null,
					"loss_fn":"BCELoss",
					"size_average":null,
					"weight":null,
					"reduction":"mean"
				},
				"optimizer":{
					"weight_decay":0.001,
					"amsgrad":false,
					"optimizer":"Adam",
					"lr":0.001,
					"betas":[
						0.9,
						0.999
					],
					"eps":1E-8,
					"config_type":"pytorch"
				},
				"nn_define":{
					"0-0":{
						"param":{},
						"module_name":"bert_",
						"class_name":"BertClassifier",
						"layer":"CustModel",
						"initializer":{}
					}
				},
				"trainer":{
					"param":{
						"batch_size":16,
						"data_loader_worker":8,
						"epochs":2
					},
					"trainer_name":"fedavg_trainer"
				},
				"dataset":{
					"param":{
						"tokenizer_name_or_path":"bert-base-uncased"
					},
					"dataset_name":"nlp_tokenizer"
				},
				"torch_seed":100
			},
			"eval_0":{
				"eval_type":"binary"
			}
		}
	},
	"dsl_version":2,
	"role":{
		"host":[
			9999
		]
	}
}

icutool avatar May 23 '24 12:05 icutool