coremltools
coremltools copied to clipboard
apple
Convert llm-like model to CoreML
I followed the guide in https://apple.github.io/coremltools/docs-guides/source/convert-openelm.html, trying to convert llm-like model to CoreML format, but encountered a bug with this line of code
mlmodel = ct.convert(exported_program)
The bug:
When both 'convert_to' and 'minimum_deployment_target' not specified, 'convert_to' is set to "mlprogram" and 'minimum_deployment_target' is set to ct.target.iOS15 (which is same as ct.target.macOS12). Note: the model will not run on systems older than iOS15/macOS12/watchOS8/tvOS15. In order to make your model run on older system, please set the 'minimum_deployment_target' to iOS14/iOS13. Details please see the link: https://apple.github.io/coremltools/docs-guides/source/target-conversion-formats.html
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[30], line 1
----> 1 mlmodel = ct.convert(
2 exported_program,
3 # convert_to="neuralnetwork",
4 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/_converters_entry.py:635, in convert(model, source, inputs, outputs, classifier_config, minimum_deployment_target, convert_to, compute_precision, skip_model_load, compute_units, package_dir, debug, pass_pipeline, states)
632 if len(states) > 0 and exact_source != "pytorch":
633 raise ValueError("'states' can only be passed with pytorch source model.")
--> 635 mlmodel = mil_convert(
636 model,
637 convert_from=exact_source,
638 convert_to=exact_target,
639 inputs=inputs,
640 outputs=outputs_as_tensor_or_image_types, # None or list[ct.ImageType/ct.TensorType]
641 classifier_config=classifier_config,
642 skip_model_load=skip_model_load,
643 compute_units=compute_units,
644 package_dir=package_dir,
645 debug=debug,
646 specification_version=specification_version,
647 main_pipeline=pass_pipeline,
648 use_default_fp16_io=use_default_fp16_io,
649 states=states,
650 )
652 if exact_target == "mlprogram" and mlmodel._input_has_infinite_upper_bound():
653 raise ValueError(
654 "For mlprogram, inputs with infinite upper_bound is not allowed. Please set upper_bound"
655 ' to a positive value in "RangeDim()" for the "inputs" param in ct.convert().'
656 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:186, in mil_convert(model, convert_from, convert_to, compute_units, **kwargs)
147 @_profile
148 def mil_convert(
149 model,
(...) 153 **kwargs
154 ):
155 """
156 Convert model from a specified frontend `convert_from` to a specified
157 converter backend `convert_to`.
(...) 184 See `coremltools.converters.convert`
185 """
--> 186 return _mil_convert(
187 model,
188 convert_from,
189 convert_to,
190 ConverterRegistry,
191 ct.models.MLModel,
192 compute_units,
193 **kwargs,
194 )
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:218, in _mil_convert(model, convert_from, convert_to, registry, modelClass, compute_units, **kwargs)
215 weights_dir = _tempfile.TemporaryDirectory()
216 kwargs["weights_dir"] = weights_dir.name
--> 218 proto, mil_program = mil_convert_to_proto(
219 model,
220 convert_from,
221 convert_to,
222 registry,
223 **kwargs
224 )
226 _reset_conversion_state()
228 if convert_to == 'milinternal':
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:294, in mil_convert_to_proto(model, convert_from, convert_to, converter_registry, main_pipeline, **kwargs)
289 frontend_pipeline, backend_pipeline = _construct_other_pipelines(
290 main_pipeline, convert_from, convert_to
291 )
293 frontend_converter = frontend_converter_type()
--> 294 prog = frontend_converter(model, **kwargs)
295 PassPipelineManager.apply_pipeline(prog, frontend_pipeline)
297 PassPipelineManager.apply_pipeline(prog, main_pipeline)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/converter.py:106, in TorchFrontend.__call__(self, *args, **kwargs)
103 def __call__(self, *args, **kwargs):
104 from .frontend.torch.load import load
--> 106 return load(*args, **kwargs)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/load.py:78, in load(spec, inputs, specification_version, debug, outputs, cut_at_symbols, use_default_fp16_io, states, **kwargs)
75 else:
76 model = _torchscript_from_spec(spec)
---> 78 converter = TorchConverter(
79 model,
80 inputs,
81 outputs,
82 cut_at_symbols,
83 specification_version,
84 use_default_fp16_io,
85 states,
86 )
88 return _perform_torch_convert(converter, debug)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/converter.py:626, in TorchConverter.__init__(self, loaded_model, inputs, outputs, cut_at_symbols, opset_version, use_default_fp16_io, states)
621 raise NotImplementedError(
622 "Conversion for models with only ATEN or EDGE dialect is supported/tested. "
623 f"Provided Dialect: {loaded_model.dialect}"
624 )
625 self.context = TranscriptionContext(frontend=frontend)
--> 626 self.graph = InternalTorchIRGraph.from_exir(
627 exir=loaded_model, cut_at_symbols=cut_at_symbols
628 )
630 # finalize inputs after internal graph gets settled
631 self.inputs = self._match_user_exir_inputs(inputs)
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/internal_graph.py:590, in InternalTorchIRGraph.from_exir(cls, exir, cut_at_symbols)
588 for node in exported_program.graph_module.graph.nodes:
589 if node.op == "call_function":
--> 590 nodes.append(InternalTorchIRNode.from_exir_node(node=node))
591 elif node.op == "get_attr":
592 name = node.target
File ~/anaconda3/envs/env-coreml/lib/python3.12/site-packages/coremltools/converters/mil/frontend/torch/internal_graph.py:289, in InternalTorchIRNode.from_exir_node(cls, node)
287 kind = sanitize_op_kind(kind)
288 if kind not in _TORCH_OPS_REGISTRY:
--> 289 raise ValueError(f"Unsupported fx node {str(node)}, kind {kind}")
291 inputs = get_arguments(node.args)
292 outputs = [node.name]
ValueError: Unsupported fx node bitwise_or, kind bitwise_or
And I also want to know when we want to run llm on apple device, do we usually convert the llm to CoreML format or there is another way?
ValueError: Unsupported fx node bitwise_or, kind bitwise_or
This means your torch model has op bitwise_or that we haven't supported yet 🙏 We would encourage to add its translation following the example on how we translate torch.logical_or
do we usually convert the llm to Core ML format or there is another way?
Yes, we do recommend to convert to Core ML format, since Core ML is available on all Apple hardware (Mac, iPhone, ...)
