我正在尝试在我自己的词典上训练现有的词性标注器，而不是从头开始（我不想创建一个“空模型”）。
在spaCy的文档中，它说“加载您想要统计的模型”，下一步是“使用add_label方法将标签映射添加到标记器”。但是，当我尝试加载英文小模型并添加标签图时，它会抛出此错误：
ValueError：[T003] 目前不支持调整预训练标记器模型的大小。
我想知道如何修复它。
我也见过在 Spacy 中基于现有英语模型实现自定义 POS Tagger：NLP - Python https://stackoverflow.com/questions/51715439/implementing-custom-pos-tagger-in-spacy-over-existing-english-model-nlp-pyth但它表明我们创建了一个“空模型”，这不是我想要的。
另外，即使我们的训练示例标签与通用依赖标签相同，spaCy 的文档中也不清楚我们是否需要映射字典（TAG_MAP）。有什么想法吗？
from __future__ import unicode_literals, print_function
import plac
import random
from pathlib import Path
import spacy
from spacy.util import minibatch, compounding
TAG_MAP = {"noun": {"pos": "NOUN"}, "verb": {"pos": "VERB"}, "adj": {"pos": "ADJ"}, "adv": {"pos": "ADV"}}
TRAIN_DATA = [
('Afrotropical', {'tags': ['adj']}), ('Afrocentricity', {'tags': ['noun']}),
('Afrocentric', {'tags': ['adj']}), ('Afrocentrism', {'tags': ['noun']}),
('Anglomania', {'tags': ['noun']}), ('Anglocentric', {'tags': ['adj']}),
('apraxic', {'tags': ['adj']}), ('aglycosuric', {'tags': ['adj']}),
('asecretory', {'tags': ['adj']}), ('aleukaemic', {'tags': ['adj']}),
('agrin', {'tags': ['adj']}), ('Eurotransplant', {'tags': ['noun']}),
('Euromarket', {'tags': ['noun']}), ('Eurocentrism', {'tags': ['noun']}),
('adendritic', {'tags': ['adj']}), ('asynaptic', {'tags': ['adj']}),
('Asynapsis', {'tags': ['noun']}), ('ametabolic', {'tags': ['adj']})
]
@plac.annotations(
lang=("ISO Code of language to use", "option", "l", str),
output_dir=("Optional output directory", "option", "o", Path),
n_iter=("Number of training iterations", "option", "n", int),
)
def main(lang="en", output_dir=None, n_iter=25):
nlp = spacy.load('en_core_web_sm', disable=['ner', 'parser'])
tagger = nlp.get_pipe('tagger')
for tag, values in TAG_MAP.items():
tagger.add_label(tag, values)
nlp.vocab.vectors.name = 'spacy_pretrained_vectors'
optimizer = nlp.begin_training()
for i in range(n_iter):
random.shuffle(TRAIN_DATA)
losses = {}
# batch up the examples using spaCy's minibatch
batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=optimizer, losses=losses)
print("Losses", losses)
# test the trained model
test_text = "I like Afrotropical apraxic blue eggs and Afrocentricity. A Eurotransplant is cool too. The agnathostomatous Euromarket and asypnapsis is even cooler. What about Eurocentrism?"
doc = nlp(test_text)
print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the save model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
doc = nlp2(test_text)
print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])
if __name__ == "__main__":
plac.call(main)
