Бинарная несовместимость с Numpy ⇐ Python

[Anonymous]

Я использую ноутбук Kaggle с графическим процессором T4 и следующий код:

Код: Выделить всё

%%capture

!pip uninstall -y transformers datasets accelerate peft huggingface_hub bitsandbytes sentence-transformers faiss-gpu

!pip install transformers accelerate peft huggingface_hub datasets
!pip install bitsandbytes==0.49.2
!pip install sentence-transformers
!pip install faiss-gpu
!pip install evaluate rouge_score bert-score

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
from huggingface_hub import login
from datasets import load_dataset
from sentence_transformers import SentenceTransformer
from sklearn.neighbors import NearestNeighbors
import numpy as np
import os
import gc
from tqdm import tqdm
import evaluate
import time

#hf_token = ""
#login(token=hf_token)

#base_model_id = "Chilliwiddit/Llama-3.1-8B-RConceptM"
base_model_id = "unsloth/Meta-Llama-3.1-8B-bnb-4bit"
#base_model_id = "meta-llama/Llama-3.1-8B"

adapter_model_id = "Chilliwiddit/llama3.1-8B-WeightedLoss-small1"

print("loading base model")
base_model = AutoModelForCausalLM.from_pretrained(
base_model_id,
device_map="auto",
low_cpu_mem_usage=True,
torch_dtype=torch.float16,
)

print("loading PEFT adapter")
model = PeftModel.from_pretrained(base_model, adapter_model_id)
model.eval()

print("and tokenizer")
tokenizer = AutoTokenizer.from_pretrained(adapter_model_id)
print("done")

!pip install -U scispacy
!pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.4/en_core_sci_lg-0.5.4.tar.gz
#!pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.4/scispacy_umls_linker-0.5.4.tar.gz
#!pip install evaluate bert_score

import spacy
from scispacy.umls_linking import UmlsEntityLinker
from scispacy.linking import EntityLinker

Импорт пробелов в финальный блок кода приводит к следующему стеку ошибок:

Код: Выделить всё

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/tmp/ipykernel_55/2092502649.py in ()
----> 1 import spacy
2 from scispacy.umls_linking import UmlsEntityLinker
3 from scispacy.linking import EntityLinker

/usr/local/lib/python3.12/dist-packages/spacy/__init__.py in 
4
5 # set library-specific custom warning handling before doing anything else
----> 6 from .errors import setup_default_warnings
7
8 setup_default_warnings()  # noqa: E402

/usr/local/lib/python3.12/dist-packages/spacy/errors.py in 
1 import warnings
2
----> 3 from .compat import Literal
4
5

/usr/local/lib/python3.12/dist-packages/spacy/compat.py in 
37     from catalogue import _importlib_metadata as importlib_metadata  # type: ignore[no-redef]    # noqa: F401
38
---> 39 from thinc.api import Optimizer  # noqa: F401
40
41 pickle = pickle

/usr/local/lib/python3.12/dist-packages/thinc/api.py in 
----> 1 from .backends import (
2     CupyOps,
3     MPSOps,
4     NumpyOps,
5     Ops,

/usr/local/lib/python3.12/dist-packages/thinc/backends/__init__.py in 
15 from ._cupy_allocators import cupy_pytorch_allocator, cupy_tensorflow_allocator
16 from ._param_server import ParamServer
---> 17 from .cupy_ops import CupyOps
18 from .mps_ops import MPSOps
19 from .numpy_ops import NumpyOps

/usr/local/lib/python3.12/dist-packages/thinc/backends/cupy_ops.py in 
14 )
15 from . import _custom_kernels
---> 16 from .numpy_ops import NumpyOps
17 from .ops import Ops
18

/usr/local/lib/python3.12/dist-packages/thinc/backends/numpy_ops.pyx in init thinc.backends.numpy_ops()

ValueError: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject

До недавнего времени блокнот Kaggle работал нормально. Как я могу определить и устранить эту проблему? Я предполагаю, что это МОЖЕТ быть связано с некоторыми обновленными пакетами, но я не уверен. Есть ли обходной путь для этого?


Подробнее здесь: https://stackoverflow.com/questions/798 ... with-numpy