PSE2_FF
/
pse2_ff
0
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

Fully working flow of arc2.

pull/34/head
s8613 2025-04-29 19:02:10 +02:00
parent d0c41ceb54
commit 7f69ac8414
38 changed files with 4480 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
prototypes/arc2_prototype/.python-version 100644
View File

@ -0,0 +1 @@
3.11.8

3
prototypes/arc2_prototype/README.md
View File

@ -2,7 +2,8 @@
### 1. Create and activate a virtual environment
<pre><code>python3 -m venv .venv
<pre><code>pyenv shell 3.11.8
python -m venv .venv
source .venv/bin/activate
</code></pre>

6
prototypes/arc2_prototype/app.py
View File

@ -3,6 +3,8 @@ import os
import json
from ocr_pdf_service.ocr_runner import run_ocr_and_extract
from exxeta_service.exxetaGPT_api import extract_with_exxeta
from spacy_service.spacy import extract_with_spacy
from merge_validate_service.merge_validate import merge_and_validate_entities
app = Flask(__name__)
UPLOAD_FOLDER = "pitchbooks"
@ -31,11 +33,13 @@ def upload():
pitchbook_pages = json.load(f)
extract_with_exxeta(pitchbook_pages)
extract_with_spacy(pitchbook_pages)
merge_and_validate_entities()
except Exception as e:
return jsonify({"error": str(e)}), 500
return "status: complete"
return "status: complete\n"
if __name__ == "__main__":
app.run(debug=True)

40
prototypes/arc2_prototype/exxeta_service/output/exxeta-results.json
View File

@ -6,7 +6,7 @@
},
{
"label": "RISIKOPROFIL",
"entity": "Core, core+, value-added",
"entity": "core, core+, value-added",
"page": 7
},
{
@ -72,6 +72,11 @@
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 19
},
{
@ -87,11 +92,11 @@
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 26
"page": 28
},
{
"label": "RISIKOPROFIL",
"entity": "Value-added",
"entity": "Core",
"page": 26
},
{
@ -101,19 +106,19 @@
},
{
"label": "RISIKOPROFIL",
"entity": "Core Offices, quality Retail asset",
"page": 27
},
{
"label": "RISIKOPROFIL",
"entity": "Core + assets",
"page": 27
"entity": "Core, Core+",
"page": 33
},
{
"label": "RISIKOPROFIL",
"entity": "Core, Core+",
"page": 33
},
{
"label": "RISIKOPROFIL",
"entity": "Value-added",
"page": 33
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
@ -138,5 +143,20 @@
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 37
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 37
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 38
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 38
}
]

71
prototypes/arc2_prototype/merge_validate_service/merge_validate.py 100644
View File

@ -0,0 +1,71 @@
import json
from pathlib import Path
def normalize_entity(entity_str):
if not entity_str:
return ""
normalized = entity_str.replace('\n', ' ')
normalized = ''.join(normalized.lower().split())
return normalized
def merge_and_validate_entities():
base_dir = Path(__file__).resolve().parent
spacy_results_path = base_dir / ".." / "spacy_service" / "output" / "spacy-results.json"
exxeta_results_path = base_dir / ".." / "exxeta_service" / "output" / "exxeta-results.json"
output_path = base_dir / "output" / "merged-results.json"
with open(spacy_results_path, "r", encoding="utf-8") as f:
spacy_data = json.load(f)
with open(exxeta_results_path, "r", encoding="utf-8") as f:
exxeta_data = json.load(f)
merged = []
seen = set()
for s in spacy_data:
s_entity_norm = normalize_entity(s["entity"])
s_page = s["page"]
found = False
for e in exxeta_data:
e_entity_norm = normalize_entity(e["entity"])
e_page = e["page"]
if (s["label"] == e["label"] and
s_entity_norm == e_entity_norm and
s_page == e_page):
merged.append({
"label": s["label"],
"entity": s["entity"],
"page": s_page,
"status": "validated"
})
seen.add((e["entity"], e_page))
found = True
break
if not found:
merged.append({
"label": s["label"],
"entity": s["entity"],
"page": s_page,
"status": "single-source",
"source": "spacy"
})
for e in exxeta_data:
if (e["entity"], e["page"]) not in seen:
merged.append({
"label": e["label"],
"entity": e["entity"],
"page": e["page"],
"status": "single-source",
"source": "exxeta"
})
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(merged, f, indent=2, ensure_ascii=False)
return merged

447
prototypes/arc2_prototype/merge_validate_service/output/merged-results.json 100644
View File

@ -0,0 +1,447 @@
[
{
"label": "RISIKOPROFIL",
"entity": "Core and Core+",
"page": 4,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "core, core+, value-added",
"page": 7,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 10,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 10,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 10,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "UK, DE, BE, NL, LU,",
"page": 10,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core / Core +",
"page": 12,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "core\n/ core+",
"page": 12,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "core\n",
"page": 12,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Term / core+",
"page": 12,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 12,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "6,4 6,4",
"page": 13,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Country /",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Excellent",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 40",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 400m",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 99m-102",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 85m-90m",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 50",
"page": 14,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Country /",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core 44m-46m",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core CBD New",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "countries, giving",
"page": 18,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 20,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 20,
"status": "validated"
},
{
"label": "RISIKOPROFIL",
"entity": "D, and",
"page": 21,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "UK, DE, BE, NL, LU,",
"page": 26,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "core or",
"page": 27,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core +",
"page": 27,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "kgCO,e",
"page": 30,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "C,\n",
"page": 32,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "KfW, Dwp",
"page": 35,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Bank,",
"page": 35,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "HSBC, RTE",
"page": 37,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core WALB (",
"page": 37,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core WALB (",
"page": 37,
"status": "single-source",
"source": "spacy"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 9,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 14,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 14,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 14,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 14,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 14,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 15,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 19,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 28,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 26,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Offices, Core + assets",
"page": 27,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core, Core+",
"page": 33,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core, Core+",
"page": 33,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Value-added",
"page": 33,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 35,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 35,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Parking",
"page": 36,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core Parking",
"page": 36,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 37,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 37,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 38,
"status": "single-source",
"source": "exxeta"
},
{
"label": "RISIKOPROFIL",
"entity": "Core",
"page": 38,
"status": "single-source",
"source": "exxeta"
}
]

BIN
prototypes/arc2_prototype/ocr_pdf_service/output/pitchbook-OCR.pdf
View File

Binary file not shown.

BIN
prototypes/arc2_prototype/pitchbooks/Pitchbook 1.pdf 100644
View File

Binary file not shown.

BIN
prototypes/arc2_prototype/pitchbooks/Pitchbook 3.pdf 100644
View File

Binary file not shown.

78
prototypes/arc2_prototype/requirements.txt
View File

@ -1,4 +1,74 @@
flask
pdfplumber
ocrmypdf
requests
annotated-types==0.7.0
blinker==1.9.0
blis==0.7.11
catalogue==2.0.10
certifi==2025.4.26
cffi==1.17.1
charset-normalizer==3.4.1
click==8.1.8
cloudpathlib==0.16.0
confection==0.1.5
cryptography==44.0.2
cymem==2.0.11
Deprecated==1.2.18
deprecation==2.1.0
filelock==3.18.0
Flask==3.1.0
fsspec==2025.3.2
huggingface-hub==0.30.2
idna==3.10
img2pdf==0.6.1
itsdangerous==2.2.0
Jinja2==3.1.6
langcodes==3.5.0
language_data==1.3.0
lxml==5.4.0
marisa-trie==1.2.1
markdown-it-py==3.0.0
MarkupSafe==3.0.2
mdurl==0.1.2
mpmath==1.3.0
murmurhash==1.0.12
networkx==3.4.2
numpy==1.26.4
ocrmypdf==16.10.1
packaging==25.0
pdfminer.six==20250327
pdfplumber==0.11.6
pi_heif==0.22.0
pikepdf==9.7.0
pillow==11.2.1
pluggy==1.5.0
preshed==3.0.9
pycparser==2.22
pydantic==2.11.3
pydantic_core==2.33.1
Pygments==2.19.1
PyMuPDF==1.25.5
pypdfium2==4.30.1
PyYAML==6.0.2
regex==2024.11.6
requests==2.32.3
rich==14.0.0
safetensors==0.5.3
smart-open==6.4.0
spacy==3.7.2
spacy-alignments==0.9.1
spacy-legacy==3.0.12
spacy-loggers==1.0.5
spacy-transformers==1.3.3
srsly==2.5.1
sympy==1.14.0
thinc==8.2.5
tokenizers==0.15.2
torch==2.1.0
tqdm==4.67.1
transformers==4.35.2
typer==0.9.4
typing-inspection==0.4.0
typing_extensions==4.13.2
urllib3==2.4.0
wasabi==1.1.3
weasel==0.3.4
Werkzeug==3.1.3
wrapt==1.17.2

145
prototypes/arc2_prototype/spacy_service/models/model-best/config.cfg 100644
View File

@ -0,0 +1,145 @@
[paths]
train = "./data/train.spacy"
dev = "./data/train.spacy"
vectors = null
init_tok2vec = null
[system]
gpu_allocator = null
seed = 0
[nlp]
lang = "de"
pipeline = ["tok2vec","ner"]
batch_size = 1000
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
vectors = {"@vectors":"spacy.Vectors.v1"}
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = true
nO = null
[components.ner.model.tok2vec]
@architectures = "spacy.Tok2VecListener.v1"
width = ${components.tok2vec.model.encode.width}
upstream = "*"
[components.tok2vec]
factory = "tok2vec"
[components.tok2vec.model]
@architectures = "spacy.Tok2Vec.v2"
[components.tok2vec.model.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = ${components.tok2vec.model.encode.width}
attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
rows = [5000,1000,2500,2500]
include_static_vectors = false
[components.tok2vec.model.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 96
depth = 4
window_size = 1
maxout_pieces = 3
[corpora]
[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
accumulate_gradient = 1
patience = 1600
max_epochs = 0
max_steps = 20000
eval_frequency = 200
frozen_components = []
annotating_components = []
before_to_disk = null
before_update = null
[training.batcher]
@batchers = "spacy.batch_by_words.v1"
discard_oversize = false
tolerance = 0.2
get_length = null
[training.batcher.size]
@schedules = "compounding.v1"
start = 100
stop = 1000
compound = 1.001
t = 0.0
[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
progress_bar = false
[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
learn_rate = 0.001
[training.score_weights]
ents_f = 1.0
ents_p = 0.0
ents_r = 0.0
ents_per_type = null
[pretraining]
[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]

52
prototypes/arc2_prototype/spacy_service/models/model-best/meta.json 100644
View File

@ -0,0 +1,52 @@
{
"lang":"de",
"name":"pipeline",
"version":"0.0.0",
"spacy_version":">=3.7.2,<3.8.0",
"description":"",
"author":"",
"email":"",
"url":"",
"license":"",
"spacy_git_version":"a89eae928",
"vectors":{
"width":0,
"vectors":0,
"keys":0,
"name":null,
"mode":"default"
},
"labels":{
"tok2vec":[
],
"ner":[
"RISIKOPROFIL"
]
},
"pipeline":[
"tok2vec",
"ner"
],
"components":[
"tok2vec",
"ner"
],
"disabled":[
],
"performance":{
"ents_f":1.0,
"ents_p":1.0,
"ents_r":1.0,
"ents_per_type":{
"RISIKOPROFIL":{
"p":1.0,
"r":1.0,
"f":1.0
}
},
"tok2vec_loss":0.000000029,
"ner_loss":0.0000000614
}
}

13
prototypes/arc2_prototype/spacy_service/models/model-best/ner/cfg 100644
View File

@ -0,0 +1,13 @@
{
"moves":null,
"update_with_oracle_cut_size":100,
"multitasks":[
],
"min_action_freq":1,
"learn_tokens":false,
"beam_width":1,
"beam_density":0.0,
"beam_update_prob":0.0,
"incorrect_spans_key":null
}

BIN
prototypes/arc2_prototype/spacy_service/models/model-best/ner/model 100644
View File

Binary file not shown.

1
prototypes/arc2_prototype/spacy_service/models/model-best/ner/moves 100644
View File

@ -0,0 +1 @@
‚ĄmovesŮx{"0":{},"1":{"RISIKOPROFIL":45},"2":{"RISIKOPROFIL":45},"3":{"RISIKOPROFIL":45},"4":{"RISIKOPROFIL":45,"":1},"5":{"":1}}Łcfg<66>§neg_keyŔ

3
prototypes/arc2_prototype/spacy_service/models/model-best/tok2vec/cfg 100644
View File

@ -0,0 +1,3 @@
{
}

BIN
prototypes/arc2_prototype/spacy_service/models/model-best/tok2vec/model 100644
View File

Binary file not shown.

3
prototypes/arc2_prototype/spacy_service/models/model-best/tokenizer 100644
View File

File diff suppressed because one or more lines are too long

1
prototypes/arc2_prototype/spacy_service/models/model-best/vocab/key2row 100644
View File

@ -0,0 +1 @@
<EFBFBD>

1
prototypes/arc2_prototype/spacy_service/models/model-best/vocab/lookups.bin 100644
View File

@ -0,0 +1 @@
<EFBFBD>

1585
prototypes/arc2_prototype/spacy_service/models/model-best/vocab/strings.json 100644
View File

File diff suppressed because it is too large Load Diff

BIN
prototypes/arc2_prototype/spacy_service/models/model-best/vocab/vectors 100644
View File

Binary file not shown.

3
prototypes/arc2_prototype/spacy_service/models/model-best/vocab/vectors.cfg 100644
View File

@ -0,0 +1,3 @@
{
"mode":"default"
}

145
prototypes/arc2_prototype/spacy_service/models/model-last/config.cfg 100644
View File

@ -0,0 +1,145 @@
[paths]
train = "./data/train.spacy"
dev = "./data/train.spacy"
vectors = null
init_tok2vec = null
[system]
gpu_allocator = null
seed = 0
[nlp]
lang = "de"
pipeline = ["tok2vec","ner"]
batch_size = 1000
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
vectors = {"@vectors":"spacy.Vectors.v1"}
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = true
nO = null
[components.ner.model.tok2vec]
@architectures = "spacy.Tok2VecListener.v1"
width = ${components.tok2vec.model.encode.width}
upstream = "*"
[components.tok2vec]
factory = "tok2vec"
[components.tok2vec.model]
@architectures = "spacy.Tok2Vec.v2"
[components.tok2vec.model.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = ${components.tok2vec.model.encode.width}
attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
rows = [5000,1000,2500,2500]
include_static_vectors = false
[components.tok2vec.model.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 96
depth = 4
window_size = 1
maxout_pieces = 3
[corpora]
[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
accumulate_gradient = 1
patience = 1600
max_epochs = 0
max_steps = 20000
eval_frequency = 200
frozen_components = []
annotating_components = []
before_to_disk = null
before_update = null
[training.batcher]
@batchers = "spacy.batch_by_words.v1"
discard_oversize = false
tolerance = 0.2
get_length = null
[training.batcher.size]
@schedules = "compounding.v1"
start = 100
stop = 1000
compound = 1.001
t = 0.0
[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
progress_bar = false
[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
learn_rate = 0.001
[training.score_weights]
ents_f = 1.0
ents_p = 0.0
ents_r = 0.0
ents_per_type = null
[pretraining]
[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]

52
prototypes/arc2_prototype/spacy_service/models/model-last/meta.json 100644
View File

@ -0,0 +1,52 @@
{
"lang":"de",
"name":"pipeline",
"version":"0.0.0",
"spacy_version":">=3.7.2,<3.8.0",
"description":"",
"author":"",
"email":"",
"url":"",
"license":"",
"spacy_git_version":"a89eae928",
"vectors":{
"width":0,
"vectors":0,
"keys":0,
"name":null,
"mode":"default"
},
"labels":{
"tok2vec":[
],
"ner":[
"RISIKOPROFIL"
]
},
"pipeline":[
"tok2vec",
"ner"
],
"components":[
"tok2vec",
"ner"
],
"disabled":[
],
"performance":{
"ents_f":1.0,
"ents_p":1.0,
"ents_r":1.0,
"ents_per_type":{
"RISIKOPROFIL":{
"p":1.0,
"r":1.0,
"f":1.0
}
},
"tok2vec_loss":0.000000029,
"ner_loss":0.0000000614
}
}

13
prototypes/arc2_prototype/spacy_service/models/model-last/ner/cfg 100644
View File

@ -0,0 +1,13 @@
{
"moves":null,
"update_with_oracle_cut_size":100,
"multitasks":[
],
"min_action_freq":1,
"learn_tokens":false,
"beam_width":1,
"beam_density":0.0,
"beam_update_prob":0.0,
"incorrect_spans_key":null
}

BIN
prototypes/arc2_prototype/spacy_service/models/model-last/ner/model 100644
View File

Binary file not shown.

1
prototypes/arc2_prototype/spacy_service/models/model-last/ner/moves 100644
View File

@ -0,0 +1 @@
‚ĄmovesŮx{"0":{},"1":{"RISIKOPROFIL":45},"2":{"RISIKOPROFIL":45},"3":{"RISIKOPROFIL":45},"4":{"RISIKOPROFIL":45,"":1},"5":{"":1}}Łcfg<66>§neg_keyŔ

3
prototypes/arc2_prototype/spacy_service/models/model-last/tok2vec/cfg 100644
View File

@ -0,0 +1,3 @@
{
}

BIN
prototypes/arc2_prototype/spacy_service/models/model-last/tok2vec/model 100644
View File

Binary file not shown.

3
prototypes/arc2_prototype/spacy_service/models/model-last/tokenizer 100644
View File

File diff suppressed because one or more lines are too long

1
prototypes/arc2_prototype/spacy_service/models/model-last/vocab/key2row 100644
View File

@ -0,0 +1 @@
<EFBFBD>

1
prototypes/arc2_prototype/spacy_service/models/model-last/vocab/lookups.bin 100644
View File

@ -0,0 +1 @@
<EFBFBD>

1585
prototypes/arc2_prototype/spacy_service/models/model-last/vocab/strings.json 100644
View File

File diff suppressed because it is too large Load Diff

BIN
prototypes/arc2_prototype/spacy_service/models/model-last/vocab/vectors 100644
View File

Binary file not shown.

3
prototypes/arc2_prototype/spacy_service/models/model-last/vocab/vectors.cfg 100644
View File

@ -0,0 +1,3 @@
{
"mode":"default"
}

197
prototypes/arc2_prototype/spacy_service/output/spacy-results.json 100644
View File

@ -0,0 +1,197 @@
[
{
"label": "RISIKOPROFIL",
"entity": "Core and Core+",
"page": 4
},
{
"label": "RISIKOPROFIL",
"entity": "core, core+, value-added",
"page": 7
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 10
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 10
},
{
"label": "RISIKOPROFIL",
"entity": "Core/Core+",
"page": 10
},
{
"label": "RISIKOPROFIL",
"entity": "UK, DE, BE, NL, LU,",
"page": 10
},
{
"label": "RISIKOPROFIL",
"entity": "Core / Core +",
"page": 12
},
{
"label": "RISIKOPROFIL",
"entity": "core\n/ core+",
"page": 12
},
{
"label": "RISIKOPROFIL",
"entity": "core\n",
"page": 12
},
{
"label": "RISIKOPROFIL",
"entity": "Term / core+",
"page": 12
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 12
},
{
"label": "RISIKOPROFIL",
"entity": "6,4 6,4",
"page": 13
},
{
"label": "RISIKOPROFIL",
"entity": "Country /",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core Excellent",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core 40",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core 400m",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core 99m-102",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core 85m-90m",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Core 50",
"page": 14
},
{
"label": "RISIKOPROFIL",
"entity": "Country /",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core 44m-46m",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core CBD New",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "Core Good New",
"page": 15
},
{
"label": "RISIKOPROFIL",
"entity": "countries, giving",
"page": 18
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 20
},
{
"label": "RISIKOPROFIL",
"entity": "core/core+",
"page": 20
},
{
"label": "RISIKOPROFIL",
"entity": "D, and",
"page": 21
},
{
"label": "RISIKOPROFIL",
"entity": "UK, DE, BE, NL, LU,",
"page": 26
},
{
"label": "RISIKOPROFIL",
"entity": "core or",
"page": 27
},
{
"label": "RISIKOPROFIL",
"entity": "Core +",
"page": 27
},
{
"label": "RISIKOPROFIL",
"entity": "kgCO,e",
"page": 30
},
{
"label": "RISIKOPROFIL",
"entity": "C,\n",
"page": 32
},
{
"label": "RISIKOPROFIL",
"entity": "KfW, Dwp",
"page": 35
},
{
"label": "RISIKOPROFIL",
"entity": "Bank,",
"page": 35
},
{
"label": "RISIKOPROFIL",
"entity": "HSBC, RTE",
"page": 37
},
{
"label": "RISIKOPROFIL",
"entity": "Core WALB (",
"page": 37
},
{
"label": "RISIKOPROFIL",
"entity": "Core WALB (",
"page": 37
}
]

39
prototypes/arc2_prototype/spacy_service/spacy.py 100644
View File

@ -0,0 +1,39 @@
import spacy
from pathlib import Path
import json
import os
import fitz
OUTPUT_FOLDER = Path(__file__).resolve().parent / "output"
OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
model_path = os.path.join(os.path.dirname(__file__), "models", "model-last")
nlp = spacy.load(model_path)
input_pdf_path = Path(__file__).resolve().parent / ".." / "ocr_pdf_service" / "output" / "pitchbook-OCR.pdf"
input_pdf = Path(input_pdf_path)
doc = fitz.open(input_pdf)
def extract_with_spacy(pages_json):
results = []
for page in pages_json:
text = page.get("text", "").strip()
page_num = page.get("page")
if not text:
continue
doc = nlp(text)
for ent in doc.ents:
results.append({
"label": ent.label_,
"entity": ent.text,
"page": page_num
})
output_path = OUTPUT_FOLDER / f"spacy-results.json"
with open(output_path, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False)
return results