cudnn error
#598
Replies: 2 comments 1 reply
mmich_machine_learning | [11/18/25 16:09:42] INFO Loading recognition model 'PP-OCRv5_server' to
immich_machine_learning | memory
immich_machine_learning | [11/18/25 16:09:42] INFO Setting execution providers to
immich_machine_learning | ['CUDAExecutionProvider', 'CPUExecutionProvider'],
immich_machine_learning | in descending order of preference
immich_machine_learning | [11/18/25 16:09:43] INFO Using engine_name: onnxruntime
immich_machine_learning | 2025-11-18 16:09:44.439850657 [E:onnxruntime:, sequential_executor.cc:516 ExecuteKernel] Non-zero status code returned while running FusedConv node. Name:'Conv.64' Status Message: /onnxruntime_src/onnxruntime/core/providers/cuda/cuda_call.cc:123 std::conditional_t<THRW, void, onnxruntime::common::Status> onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*, const char*, int) [with ERRTYPE = cudaError; bool THRW = true; std::conditional_t<THRW, void, common::Status> = void] /onnxruntime_src/onnxruntime/core/providers/cuda/cuda_call.cc:116 std::conditional_t<THRW, void, onnxruntime::common::Status> onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*, const char*, int) [with ERRTYPE = cudaError; bool THRW = true; std::conditional_t<THRW, void, common::Status> = void] CUDA failure 2: out of memory ; GPU=0 ; hostname=9e765fe306d6 ; file=/onnxruntime_src/onnxruntime/core/providers/cuda/cuda_allocator.cc ; line=47 ; expr=cudaMalloc((void**)&p, size);
immich_machine_learning |
immich_machine_learning |
immich_machine_learning | [11/18/25 16:09:44] ERROR Exception in ASGI application
immich_machine_learning |
immich_machine_learning | ╭─────── Traceback (most recent call last) ───────╮
immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/rapidocr │
immich_machine_learning | │ /inference_engine/onnxruntime/main.py:90 in │
immich_machine_learning | │ __call__ │
immich_machine_learning | │ │
immich_machine_learning | │ 87 │ def __call__(self, input_content: np. │
immich_machine_learning | │ 88 │ │ input_dict = dict(zip(self.get_in │
immich_machine_learning | │ 89 │ │ try: │
immich_machine_learning | │ ❱ 90 │ │ │ return self.session.run(self. │
immich_machine_learning | │ 91 │ │ except Exception as e: │
immich_machine_learning | │ 92 │ │ │ error_info = traceback.format │
immich_machine_learning | │ 93 │ │ │ raise ONNXRuntimeError(error_ │
immich_machine_learning | │ │
immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/onnxrunt │
immich_machine_learning | │ ime/capi/onnxruntime_inference_collection.py:22 │
immich_machine_learning | │ 0 in run │
immich_machine_learning | │ │
immich_machine_learning | │ 217 │ │ if not output_names: │
immich_machine_learning | │ 218 │ │ │ output_names = [output.name │
immich_machine_learning | │ 219 │ │ try: │
immich_machine_learning | │ ❱ 220 │ │ │ return self._sess.run(output │
immich_machine_learning | │ 221 │ │ except C.EPFail as err: │
immich_machine_learning | │ 222 │ │ │ if self._enable_fallback: │
immich_machine_learning | │ 223 │ │ │ │ print(f"EP Error: {err!s │
immich_machine_learning | ╰─────────────────────────────────────────────────╯
immich_machine_learning | RuntimeException: [ONNXRuntimeError] : 6 :
immich_machine_learning | RUNTIME_EXCEPTION : Non-zero status code returned
immich_machine_learning | while running FusedConv node. Name:'Conv.64' Status
immich_machine_learning | Message:
immich_machine_learning | /onnxruntime_src/onnxruntime/core/providers/cuda/cu
immich_machine_learning | da_call.cc:123 std::conditional_t<THRW, void,
immich_machine_learning | onnxruntime::common::Status>
immich_machine_learning | onnxruntime::CudaCall(ERRTYPE, const char*, const
immich_machine_learning | char*, ERRTYPE, const char*, const char*, int)
immich_machine_learning | [with ERRTYPE = cudaError; bool THRW = true;
immich_machine_learning | std::conditional_t<THRW, void, common::Status> =
immich_machine_learning | void]
immich_machine_learning | /onnxruntime_src/onnxruntime/core/providers/cuda/cu
immich_machine_learning | da_call.cc:116 std::conditional_t<THRW, void,
immich_machine_learning | onnxruntime::common::Status>
immich_machine_learning | onnxruntime::CudaCall(ERRTYPE, const char*, const
immich_machine_learning | char*, ERRTYPE, const char*, const char*, int)
immich_machine_learning | [with ERRTYPE = cudaError; bool THRW = true;
immich_machine_learning | std::conditional_t<THRW, void, common::Status> =
immich_machine_learning | void] CUDA failure 2: out of memory ; GPU=0 ;
immich_machine_learning | hostname=9e765fe306d6 ;
immich_machine_learning | file=/onnxruntime_src/onnxruntime/core/providers/cu
immich_machine_learning | da/cuda_allocator.cc ; line=47 ;
immich_machine_learning | expr=cudaMalloc((void**)&p, size);
immich_machine_learning |
immich_machine_learning |
immich_machine_learning |
immich_machine_learning | The above exception was the direct cause of the
immich_machine_learning | following exception:
immich_machine_learning |
immich_machine_learning | ╭─────── Traceback (most recent call last) ───────╮
immich_machine_learning | │ /usr/src/immich_ml/main.py:177 in predict │
immich_machine_learning | │ │
immich_machine_learning | │ 174 │ │ inputs = text │
immich_machine_learning | │ 175 │ else: │
immich_machine_learning | │ 176 │ │ raise HTTPException(400, "Either │
immich_machine_learning | │ ❱ 177 │ response = await run_inference(inputs │
immich_machine_learning | │ 178 │ return ORJSONResponse(response) │
immich_machine_learning | │ 179 │
immich_machine_learning | │ 180 │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/src/immich_ml/main.py:204 in run_inference │
immich_machine_learning | │ │
immich_machine_learning | │ 201 │ without_deps, with_deps = entries │
immich_machine_learning | │ 202 │ await asyncio.gather(*[_run_inference │
immich_machine_learning | │ 203 │ if with_deps: │
immich_machine_learning | │ ❱ 204 │ │ await asyncio.gather(*[_run_infer │
immich_machine_learning | │ 205 │ if isinstance(payload, Image): │
immich_machine_learning | │ 206 │ │ response["imageHeight"], response │
immich_machine_learning | │ 207 │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/src/immich_ml/main.py:197 in │
immich_machine_learning | │ _run_inference │
immich_machine_learning | │ │
immich_machine_learning | │ 194 │ │ │ │ message = f"Task {entry[' │
immich_machine_learning | │ output of {dep}" │
immich_machine_learning | │ 195 │ │ │ │ raise HTTPException(400, │
immich_machine_learning | │ 196 │ │ model = await load(model) │
immich_machine_learning | │ ❱ 197 │ │ output = await run(model.predict, │
immich_machine_learning | │ 198 │ │ outputs[model.identity] = output │
immich_machine_learning | │ 199 │ │ response[entry["task"]] = output │
immich_machine_learning | │ 200 │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/src/immich_ml/main.py:215 in run │
immich_machine_learning | │ │
immich_machine_learning | │ 212 │ if thread_pool is None: │
immich_machine_learning | │ 213 │ │ return func(*args, **kwargs) │
immich_machine_learning | │ 214 │ partial_func = partial(func, *args, * │
immich_machine_learning | │ ❱ 215 │ return await asyncio.get_running_loop │
immich_machine_learning | │ 216 │
immich_machine_learning | │ 217 │
immich_machine_learning | │ 218 async def load(model: InferenceModel) -> │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/local/lib/python3.11/concurrent/futures/th │
immich_machine_learning | │ read.py:58 in run │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/src/immich_ml/models/base.py:60 in predict │
immich_machine_learning | │ │
immich_machine_learning | │ 57 │ │ self.load() │
immich_machine_learning | │ 58 │ │ if model_kwargs: │
immich_machine_learning | │ 59 │ │ │ self.configure(**model_kwargs │
immich_machine_learning | │ ❱ 60 │ │ return self._predict(*inputs) │
immich_machine_learning | │ 61 │ │
immich_machine_learning | │ 62 │ @abstractmethod │
immich_machine_learning | │ 63 │ def _predict(self, *inputs: Any, **mo │
immich_machine_learning | │ │
immich_machine_learning | │ /usr/src/immich_ml/models/ocr/recognition.py:72 │
immich_machine_learning | │ in _predict │
immich_machine_learning | │ │
immich_machine_learning | │ 69 │ │ boxes, box_scores = texts["boxes" │
immich_machine_learning | │ 70 │ │ if boxes.shape[0] == 0: │
immich_machine_learning | │ 71 │ │ │ return self._empty │
immich_machine_learning | │ ❱ 72 │ │ rec = self.model(TextRecInput(img │
immich_machine_learning | │ 73 │ │ if rec.txts is None: │
immich_machine_learning | │ 74 │ │ │ return self._empty │
immich_machine_learning | │ 75 │
immich_machine_learning | │ │
immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/rapidocr │
immich_machine_learning | │ /ch_ppocr_rec/main.py:121 in __call__ │
immich_machine_learning | │ │
immich_machine_learning | │ 118 │ │ │ │ norm_img_batch.append(nor │
immich_machine_learning | │ 119 │ │ │ norm_img_batch = np.concatena │
immich_machine_learning | │ 120 │ │ │ │
immich_machine_learning | │ ❱ 121 │ │ │ preds = self.session(norm_img │
immich_machine_learning | │ 122 │ │ │ line_results, word_results = │
immich_machine_learning | │ 123 │ │ │ │ preds, │
immich_machine_learning | │ 124 │ │ │ │ return_word_box, │
immich_machine_learning | │ │
immich_machine_learning | │ /opt/venv/lib/python3.11/site-packages/rapidocr │
immich_machine_learning | │ /inference_engine/onnxruntime/main.py:93 in │
immich_machine_learning | │ __call__ │
immich_machine_learning | │ │
immich_machine_learning | │ 90 │ │ │ return self.session.run(self. │
immich_machine_learning | │ 91 │ │ except Exception as e: │
immich_machine_learning | │ 92 │ │ │ error_info = traceback.format │
immich_machine_learning | │ ❱ 93 │ │ │ raise ONNXRuntimeError(error_ │
immich_machine_learning | │ 94 │ │
immich_machine_learning | │ 95 │ def get_input_names(self) -> List[str │
immich_machine_learning | │ 96 │ │ return [v.name for v in self.sess │
immich_machine_learning | ╰─────────────────────────────────────────────────╯
immich_machine_learning | ONNXRuntimeError: Traceback (most recent call
immich_machine_learning | last):
immich_machine_learning | File
immich_machine_learning | "/opt/venv/lib/python3.11/site-packages/rapidocr/in
immich_machine_learning | ference_engine/onnxruntime/main.py", line 90, in
immich_machine_learning | __call__
immich_machine_learning | return
immich_machine_learning | self.session.run(self.get_output_names(),
immich_machine_learning | input_dict)[0]
immich_machine_learning | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
immich_machine_learning | ^^^^^^^^^^^^^
immich_machine_learning | File
immich_machine_learning | "/opt/venv/lib/python3.11/site-packages/onnxruntime
immich_machine_learning | /capi/onnxruntime_inference_collection.py", line
immich_machine_learning | 220, in run
immich_machine_learning | return self._sess.run(output_names, input_feed,
immich_machine_learning | run_options)
immich_machine_learning | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
immich_machine_learning | ^^^^^^^^^^^^^
immich_machine_learning | onnxruntime.capi.onnxruntime_pybind11_state.Runtime
immich_machine_learning | Exception: [ONNXRuntimeError] : 6 :
immich_machine_learning | RUNTIME_EXCEPTION : Non-zero status code returned
immich_machine_learning | while running FusedConv node. Name:'Conv.64' Status
immich_machine_learning | Message:
immich_machine_learning | /onnxruntime_src/onnxruntime/core/providers/cuda/cu
immich_machine_learning | da_call.cc:123 std::conditional_t<THRW, void,
immich_machine_learning | onnxruntime::common::Status>
immich_machine_learning | onnxruntime::CudaCall(ERRTYPE, const char*, const
immich_machine_learning | char*, ERRTYPE, const char*, const char*, int)
immich_machine_learning | [with ERRTYPE = cudaError; bool THRW = true;
immich_machine_learning | std::conditional_t<THRW, void, common::Status> =
immich_machine_learning | void]
immich_machine_learning | /onnxruntime_src/onnxruntime/core/providers/cuda/cu
immich_machine_learning | da_call.cc:116 std::conditional_t<THRW, void,
immich_machine_learning | onnxruntime::common::Status>
immich_machine_learning | onnxruntime::CudaCall(ERRTYPE, const char*, const
immich_machine_learning | char*, ERRTYPE, const char*, const char*, int)
immich_machine_learning | [with ERRTYPE = cudaError; bool THRW = true;
immich_machine_learning | std::conditional_t<THRW, void, common::Status> =
immich_machine_learning | void] CUDA failure 2: out of memory ; GPU=0 ;
immich_machine_learning | hostname=9e765fe306d6 ;
immich_machine_learning | file=/onnxruntime_src/onnxruntime/core/providers/cu
immich_machine_learning | da/cuda_allocator.cc ; line=47 ;
immich_machine_learning | expr=cudaMalloc((void**)&p, size);
immich_machine_learning | |
0 replies
|
I found that it seems to be related to det_limit_side_len; what is its allowable range? |
1 reply
|
This parameter does not have a definite range of values. It is recommended that the minimun be 736 and the maximum not exceed 4000. Attention: this 4000 is based on my experience and is for reference only. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Uh oh!
There was an error while loading. Please reload this page.
问题描述 / Problem Description
When using this Python library in Immich, a critical error occurs.
运行环境 / Runtime Environment
docker cuda13
复现代码 / Reproduction Code
可能解决方案 / Possible solutions
All reactions