Fully working flow of arc2.

prototypes/arc2_prototype/.python-version

@@ -0,0 +1 @@
+3.11.8

prototypes/arc2_prototype/README.md

@@ -2,7 +2,8 @@
 
 ### 1. Create and activate a virtual environment
 
-<pre><code>python3 -m venv .venv
+<pre><code>pyenv shell 3.11.8
+python -m venv .venv
 source .venv/bin/activate
 </code></pre>
 

prototypes/arc2_prototype/app.py

@@ -3,6 +3,8 @@ import os
 import json
 from ocr_pdf_service.ocr_runner import run_ocr_and_extract
 from exxeta_service.exxetaGPT_api import extract_with_exxeta
+from spacy_service.spacy import extract_with_spacy
+from merge_validate_service.merge_validate import merge_and_validate_entities
 
 app = Flask(__name__)
 UPLOAD_FOLDER = "pitchbooks"
@@ -31,11 +33,13 @@ def upload():
             pitchbook_pages = json.load(f)
 
         extract_with_exxeta(pitchbook_pages)
+        extract_with_spacy(pitchbook_pages)
+        merge_and_validate_entities()
 
     except Exception as e:
         return jsonify({"error": str(e)}), 500
 
-    return "status: complete"
+    return "status: complete\n"
 
 if __name__ == "__main__":
     app.run(debug=True)

prototypes/arc2_prototype/exxeta_service/output/exxeta-results.json

@@ -6,7 +6,7 @@
   },
   {
     "label": "RISIKOPROFIL",
-    "entity": "Core, core+, value-added",
+    "entity": "core, core+, value-added",
     "page": 7
   },
   {
@@ -72,6 +72,11 @@
   {
     "label": "RISIKOPROFIL",
     "entity": "Core",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
     "page": 19
   },
   {
@@ -87,11 +92,11 @@
   {
     "label": "RISIKOPROFIL",
     "entity": "Core",
-    "page": 26
+    "page": 28
   },
   {
     "label": "RISIKOPROFIL",
-    "entity": "Value-added",
+    "entity": "Core",
     "page": 26
   },
   {
@@ -101,19 +106,19 @@
   },
   {
     "label": "RISIKOPROFIL",
-    "entity": "Core Offices, quality Retail asset",
+    "entity": "Core, Core+",
-    "page": 27
+    "page": 33
-  },
-  {
-    "label": "RISIKOPROFIL",
-    "entity": "Core + assets",
-    "page": 27
   },
   {
     "label": "RISIKOPROFIL",
     "entity": "Core, Core+",
     "page": 33
   },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Value-added",
+    "page": 33
+  },
   {
     "label": "RISIKOPROFIL",
     "entity": "Core",
@@ -138,5 +143,20 @@
     "label": "RISIKOPROFIL",
     "entity": "Core",
     "page": 37
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 37
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 38
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 38
   }
 ]

prototypes/arc2_prototype/merge_validate_service/merge_validate.py

@@ -0,0 +1,71 @@
+import json
+from pathlib import Path
+
+def normalize_entity(entity_str):
+    if not entity_str:
+        return ""
+    normalized = entity_str.replace('\n', ' ')
+    normalized = ''.join(normalized.lower().split())
+    return normalized
+
+def merge_and_validate_entities():
+    base_dir = Path(__file__).resolve().parent
+    spacy_results_path = base_dir / ".." / "spacy_service" / "output" / "spacy-results.json"
+    exxeta_results_path = base_dir / ".." / "exxeta_service" / "output" / "exxeta-results.json"
+    output_path = base_dir / "output" / "merged-results.json"
+
+    with open(spacy_results_path, "r", encoding="utf-8") as f:
+        spacy_data = json.load(f)
+    with open(exxeta_results_path, "r", encoding="utf-8") as f:
+        exxeta_data = json.load(f)
+
+    merged = []
+    seen = set()
+
+    for s in spacy_data:
+        s_entity_norm = normalize_entity(s["entity"])
+        s_page = s["page"]
+
+        found = False
+        for e in exxeta_data:
+            e_entity_norm = normalize_entity(e["entity"])
+            e_page = e["page"]
+
+            if (s["label"] == e["label"] and
+                    s_entity_norm == e_entity_norm and
+                    s_page == e_page):
+
+                merged.append({
+                    "label": s["label"],
+                    "entity": s["entity"],
+                    "page": s_page,
+                    "status": "validated"
+                })
+                seen.add((e["entity"], e_page))
+                found = True
+                break
+
+        if not found:
+            merged.append({
+                "label": s["label"],
+                "entity": s["entity"],
+                "page": s_page,
+                "status": "single-source",
+                "source": "spacy"
+            })
+
+    for e in exxeta_data:
+        if (e["entity"], e["page"]) not in seen:
+            merged.append({
+                "label": e["label"],
+                "entity": e["entity"],
+                "page": e["page"],
+                "status": "single-source",
+                "source": "exxeta"
+            })
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(merged, f, indent=2, ensure_ascii=False)
+
+    return merged

prototypes/arc2_prototype/merge_validate_service/output/merged-results.json

@@ -0,0 +1,447 @@
+[
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core and Core+",
+    "page": 4,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core, core+, value-added",
+    "page": 7,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
+    "page": 10,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 10,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
+    "page": 10,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "UK, DE, BE, NL, LU,",
+    "page": 10,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core / Core +",
+    "page": 12,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core\n/ core+",
+    "page": 12,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core\n",
+    "page": 12,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Term / core+",
+    "page": 12,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 12,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "6,4 6,4",
+    "page": 13,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Country /",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Excellent",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 40",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 400m",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 99m-102",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 85m-90m",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 50",
+    "page": 14,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Country /",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 44m-46m",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core CBD New",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "countries, giving",
+    "page": 18,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 20,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 20,
+    "status": "validated"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "D, and",
+    "page": 21,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "UK, DE, BE, NL, LU,",
+    "page": 26,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core or",
+    "page": 27,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core +",
+    "page": 27,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "kgCO,e",
+    "page": 30,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "C,\n",
+    "page": 32,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "KfW, Dwp",
+    "page": 35,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Bank,",
+    "page": 35,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "HSBC, RTE",
+    "page": 37,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core WALB (",
+    "page": 37,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core WALB (",
+    "page": 37,
+    "status": "single-source",
+    "source": "spacy"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 9,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 14,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 14,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 14,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 14,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 14,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 15,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 15,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 15,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 15,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 15,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
+    "page": 19,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 28,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 26,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Offices, Core + assets",
+    "page": 27,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core, Core+",
+    "page": 33,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core, Core+",
+    "page": 33,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Value-added",
+    "page": 33,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 35,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 35,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Parking",
+    "page": 36,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Parking",
+    "page": 36,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 37,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 37,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 38,
+    "status": "single-source",
+    "source": "exxeta"
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core",
+    "page": 38,
+    "status": "single-source",
+    "source": "exxeta"
+  }
+]

prototypes/arc2_prototype/requirements.txt

@@ -1,4 +1,74 @@
-flask
+annotated-types==0.7.0
-pdfplumber
+blinker==1.9.0
-ocrmypdf
+blis==0.7.11
-requests
+catalogue==2.0.10
+certifi==2025.4.26
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+cloudpathlib==0.16.0
+confection==0.1.5
+cryptography==44.0.2
+cymem==2.0.11
+Deprecated==1.2.18
+deprecation==2.1.0
+filelock==3.18.0
+Flask==3.1.0
+fsspec==2025.3.2
+huggingface-hub==0.30.2
+idna==3.10
+img2pdf==0.6.1
+itsdangerous==2.2.0
+Jinja2==3.1.6
+langcodes==3.5.0
+language_data==1.3.0
+lxml==5.4.0
+marisa-trie==1.2.1
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+mpmath==1.3.0
+murmurhash==1.0.12
+networkx==3.4.2
+numpy==1.26.4
+ocrmypdf==16.10.1
+packaging==25.0
+pdfminer.six==20250327
+pdfplumber==0.11.6
+pi_heif==0.22.0
+pikepdf==9.7.0
+pillow==11.2.1
+pluggy==1.5.0
+preshed==3.0.9
+pycparser==2.22
+pydantic==2.11.3
+pydantic_core==2.33.1
+Pygments==2.19.1
+PyMuPDF==1.25.5
+pypdfium2==4.30.1
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+safetensors==0.5.3
+smart-open==6.4.0
+spacy==3.7.2
+spacy-alignments==0.9.1
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy-transformers==1.3.3
+srsly==2.5.1
+sympy==1.14.0
+thinc==8.2.5
+tokenizers==0.15.2
+torch==2.1.0
+tqdm==4.67.1
+transformers==4.35.2
+typer==0.9.4
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+urllib3==2.4.0
+wasabi==1.1.3
+weasel==0.3.4
+Werkzeug==3.1.3
+wrapt==1.17.2

prototypes/arc2_prototype/spacy_service/models/model-best/config.cfg

@@ -0,0 +1,145 @@
+[paths]
+train = "./data/train.spacy"
+dev = "./data/train.spacy"
+vectors = null
+init_tok2vec = null
+
+[system]
+gpu_allocator = null
+seed = 0
+
+[nlp]
+lang = "de"
+pipeline = ["tok2vec","ner"]
+batch_size = 1000
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+vectors = {"@vectors":"spacy.Vectors.v1"}
+
+[components]
+
+[components.ner]
+factory = "ner"
+incorrect_spans_key = null
+moves = null
+scorer = {"@scorers":"spacy.ner_scorer.v1"}
+update_with_oracle_cut_size = 100
+
+[components.ner.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+nO = null
+
+[components.ner.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+upstream = "*"
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
+rows = [5000,1000,2500,2500]
+include_static_vectors = false
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+accumulate_gradient = 1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+annotating_components = []
+before_to_disk = null
+before_update = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+get_length = null
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+t = 0.0
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+learn_rate = 0.001
+
+[training.score_weights]
+ents_f = 1.0
+ents_p = 0.0
+ents_r = 0.0
+ents_per_type = null
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

prototypes/arc2_prototype/spacy_service/models/model-best/meta.json

@@ -0,0 +1,52 @@
+{
+  "lang":"de",
+  "name":"pipeline",
+  "version":"0.0.0",
+  "spacy_version":">=3.7.2,<3.8.0",
+  "description":"",
+  "author":"",
+  "email":"",
+  "url":"",
+  "license":"",
+  "spacy_git_version":"a89eae928",
+  "vectors":{
+    "width":0,
+    "vectors":0,
+    "keys":0,
+    "name":null,
+    "mode":"default"
+  },
+  "labels":{
+    "tok2vec":[
+
+    ],
+    "ner":[
+      "RISIKOPROFIL"
+    ]
+  },
+  "pipeline":[
+    "tok2vec",
+    "ner"
+  ],
+  "components":[
+    "tok2vec",
+    "ner"
+  ],
+  "disabled":[
+
+  ],
+  "performance":{
+    "ents_f":1.0,
+    "ents_p":1.0,
+    "ents_r":1.0,
+    "ents_per_type":{
+      "RISIKOPROFIL":{
+        "p":1.0,
+        "r":1.0,
+        "f":1.0
+      }
+    },
+    "tok2vec_loss":0.000000029,
+    "ner_loss":0.0000000614
+  }
+}

prototypes/arc2_prototype/spacy_service/models/model-best/ner/cfg

@@ -0,0 +1,13 @@
+{
+  "moves":null,
+  "update_with_oracle_cut_size":100,
+  "multitasks":[
+
+  ],
+  "min_action_freq":1,
+  "learn_tokens":false,
+  "beam_width":1,
+  "beam_density":0.0,
+  "beam_update_prob":0.0,
+  "incorrect_spans_key":null
+}

prototypes/arc2_prototype/spacy_service/models/model-best/ner/moves

@@ -0,0 +1 @@
+‚ĄmovesŮx{"0":{},"1":{"RISIKOPROFIL":45},"2":{"RISIKOPROFIL":45},"3":{"RISIKOPROFIL":45},"4":{"RISIKOPROFIL":45,"":1},"5":{"":1}}Łcfg<66>§neg_keyŔ

prototypes/arc2_prototype/spacy_service/models/model-best/tok2vec/cfg

@@ -0,0 +1,3 @@
+{
+
+}

prototypes/arc2_prototype/spacy_service/models/model-best/vocab/key2row

@@ -0,0 +1 @@
+<EFBFBD>

prototypes/arc2_prototype/spacy_service/models/model-best/vocab/lookups.bin

@@ -0,0 +1 @@
+<EFBFBD>

prototypes/arc2_prototype/spacy_service/models/model-best/vocab/vectors.cfg

@@ -0,0 +1,3 @@
+{
+  "mode":"default"
+}

prototypes/arc2_prototype/spacy_service/models/model-last/config.cfg

@@ -0,0 +1,145 @@
+[paths]
+train = "./data/train.spacy"
+dev = "./data/train.spacy"
+vectors = null
+init_tok2vec = null
+
+[system]
+gpu_allocator = null
+seed = 0
+
+[nlp]
+lang = "de"
+pipeline = ["tok2vec","ner"]
+batch_size = 1000
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+vectors = {"@vectors":"spacy.Vectors.v1"}
+
+[components]
+
+[components.ner]
+factory = "ner"
+incorrect_spans_key = null
+moves = null
+scorer = {"@scorers":"spacy.ner_scorer.v1"}
+update_with_oracle_cut_size = 100
+
+[components.ner.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+nO = null
+
+[components.ner.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+upstream = "*"
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
+rows = [5000,1000,2500,2500]
+include_static_vectors = false
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+accumulate_gradient = 1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+annotating_components = []
+before_to_disk = null
+before_update = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+get_length = null
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+t = 0.0
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+learn_rate = 0.001
+
+[training.score_weights]
+ents_f = 1.0
+ents_p = 0.0
+ents_r = 0.0
+ents_per_type = null
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

prototypes/arc2_prototype/spacy_service/models/model-last/meta.json

@@ -0,0 +1,52 @@
+{
+  "lang":"de",
+  "name":"pipeline",
+  "version":"0.0.0",
+  "spacy_version":">=3.7.2,<3.8.0",
+  "description":"",
+  "author":"",
+  "email":"",
+  "url":"",
+  "license":"",
+  "spacy_git_version":"a89eae928",
+  "vectors":{
+    "width":0,
+    "vectors":0,
+    "keys":0,
+    "name":null,
+    "mode":"default"
+  },
+  "labels":{
+    "tok2vec":[
+
+    ],
+    "ner":[
+      "RISIKOPROFIL"
+    ]
+  },
+  "pipeline":[
+    "tok2vec",
+    "ner"
+  ],
+  "components":[
+    "tok2vec",
+    "ner"
+  ],
+  "disabled":[
+
+  ],
+  "performance":{
+    "ents_f":1.0,
+    "ents_p":1.0,
+    "ents_r":1.0,
+    "ents_per_type":{
+      "RISIKOPROFIL":{
+        "p":1.0,
+        "r":1.0,
+        "f":1.0
+      }
+    },
+    "tok2vec_loss":0.000000029,
+    "ner_loss":0.0000000614
+  }
+}

prototypes/arc2_prototype/spacy_service/models/model-last/ner/cfg

@@ -0,0 +1,13 @@
+{
+  "moves":null,
+  "update_with_oracle_cut_size":100,
+  "multitasks":[
+
+  ],
+  "min_action_freq":1,
+  "learn_tokens":false,
+  "beam_width":1,
+  "beam_density":0.0,
+  "beam_update_prob":0.0,
+  "incorrect_spans_key":null
+}

prototypes/arc2_prototype/spacy_service/models/model-last/ner/moves

@@ -0,0 +1 @@
+‚ĄmovesŮx{"0":{},"1":{"RISIKOPROFIL":45},"2":{"RISIKOPROFIL":45},"3":{"RISIKOPROFIL":45},"4":{"RISIKOPROFIL":45,"":1},"5":{"":1}}Łcfg<66>§neg_keyŔ

prototypes/arc2_prototype/spacy_service/models/model-last/tok2vec/cfg

@@ -0,0 +1,3 @@
+{
+
+}

prototypes/arc2_prototype/spacy_service/models/model-last/vocab/key2row

@@ -0,0 +1 @@
+<EFBFBD>

prototypes/arc2_prototype/spacy_service/models/model-last/vocab/lookups.bin

@@ -0,0 +1 @@
+<EFBFBD>

prototypes/arc2_prototype/spacy_service/models/model-last/vocab/vectors.cfg

@@ -0,0 +1,3 @@
+{
+  "mode":"default"
+}

prototypes/arc2_prototype/spacy_service/output/spacy-results.json

@@ -0,0 +1,197 @@
+[
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core and Core+",
+    "page": 4
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core, core+, value-added",
+    "page": 7
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
+    "page": 10
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 10
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core/Core+",
+    "page": 10
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "UK, DE, BE, NL, LU,",
+    "page": 10
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core / Core +",
+    "page": 12
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core\n/ core+",
+    "page": 12
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core\n",
+    "page": 12
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Term / core+",
+    "page": 12
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 12
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "6,4 6,4",
+    "page": 13
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Country /",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Excellent",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 40",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 400m",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 99m-102",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 85m-90m",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 50",
+    "page": 14
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Country /",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core 44m-46m",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core CBD New",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core Good New",
+    "page": 15
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "countries, giving",
+    "page": 18
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 20
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core/core+",
+    "page": 20
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "D, and",
+    "page": 21
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "UK, DE, BE, NL, LU,",
+    "page": 26
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "core or",
+    "page": 27
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core +",
+    "page": 27
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "kgCO,e",
+    "page": 30
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "C,\n",
+    "page": 32
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "KfW, Dwp",
+    "page": 35
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Bank,",
+    "page": 35
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "HSBC, RTE",
+    "page": 37
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core WALB (",
+    "page": 37
+  },
+  {
+    "label": "RISIKOPROFIL",
+    "entity": "Core WALB (",
+    "page": 37
+  }
+]

prototypes/arc2_prototype/spacy_service/spacy.py

@@ -0,0 +1,39 @@
+import spacy
+from pathlib import Path
+import json
+import os
+import fitz
+
+OUTPUT_FOLDER = Path(__file__).resolve().parent / "output"
+OUTPUT_FOLDER.mkdir(parents=True, exist_ok=True)
+
+model_path = os.path.join(os.path.dirname(__file__), "models", "model-last")
+nlp = spacy.load(model_path)
+input_pdf_path = Path(__file__).resolve().parent / ".." / "ocr_pdf_service" / "output" / "pitchbook-OCR.pdf"
+input_pdf = Path(input_pdf_path)
+doc = fitz.open(input_pdf)
+
+
+def extract_with_spacy(pages_json):
+    results = []
+
+    for page in pages_json:
+        text = page.get("text", "").strip()
+        page_num = page.get("page")
+
+        if not text:
+            continue
+
+        doc = nlp(text)
+        for ent in doc.ents:
+            results.append({
+                "label": ent.label_,
+                "entity": ent.text,
+                "page": page_num
+            })
+
+    output_path = OUTPUT_FOLDER / f"spacy-results.json"
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+
+    return results