359 lines
13 KiB
Plaintext
359 lines
13 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "ad994162",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import csv\n",
|
||
"import json\n",
|
||
"import requests as rq\n",
|
||
"import bs4\n",
|
||
"import pandas as pd\n",
|
||
"import time\n",
|
||
"import random\n",
|
||
"import re\n",
|
||
"import pprint"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "b5536e8c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"producers = [\"bluebrixx\", \"cada\", \"cobi\", \"mouldking\", \"pantasy\"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a5daea73",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"ename": "FileNotFoundError",
|
||
"evalue": "[Errno 2] No such file or directory: 'bluebrixx.json'",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||
"\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[11]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m open(\u001b[33m\"./others.csv\"\u001b[39m, mode=\u001b[33m\"w+\"\u001b[39m, encoding=\u001b[33m\"utf8\"\u001b[39m, newline=\u001b[33m\"\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m producerfile:\n\u001b[32m 2\u001b[39m writer = csv.writer(producerfile)\n\u001b[32m 3\u001b[39m writer.writerow([\u001b[33m\"id\"\u001b[39m, \u001b[33m\"producer\"\u001b[39m, \u001b[33m\"name\"\u001b[39m, \u001b[33m\"size\"\u001b[39m, \u001b[33m\"parts\"\u001b[39m, \u001b[33m\"year\"\u001b[39m])\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m producer \u001b[38;5;28;01min\u001b[39;00m producers:\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m open(\u001b[33mf\"{producer}.json\"\u001b[39m, mode=\u001b[33m\"r\"\u001b[39m, encoding=\u001b[33m\"utf8\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m sourcefile:\n\u001b[32m 6\u001b[39m data = json.loads(sourcefile.read())\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m row \u001b[38;5;28;01min\u001b[39;00m data[\u001b[33m\"data\"\u001b[39m]:\n\u001b[32m 8\u001b[39m _, id, _, name, rating, _, _, size, parts, year, _ = row\n",
|
||
"\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: 'bluebrixx.json'"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"with open(\"./others.csv\", mode=\"w+\", encoding=\"utf8\", newline=\"\") as producerfile:\n",
|
||
" writer = csv.writer(producerfile)\n",
|
||
" writer.writerow([\"id\", \"producer\", \"name\", \"size\", \"parts\", \"year\"])\n",
|
||
" for producer in producers:\n",
|
||
" with open(f\"data/merlin/{producer}.json\", mode=\"r\", encoding=\"utf8\") as sourcefile:\n",
|
||
" data = json.loads(sourcefile.read())\n",
|
||
" for row in data[\"data\"]:\n",
|
||
" _, id, _, name, rating, _, _, size, parts, year, _ = row\n",
|
||
"\n",
|
||
" writer.writerow([id, producer, name, size, parts, year])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 16,
|
||
"id": "ab997198",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# uvp preise bestimmen :(\n",
|
||
"def get_all_ids() -> list[str]:\n",
|
||
" df = pd.read_csv(\"./others.csv\")\n",
|
||
" return df[\"id\"].to_list()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "32b1fa46",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"ename": "KeyboardInterrupt",
|
||
"evalue": "",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||
"\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[18]\u001b[39m\u001b[32m, line 26\u001b[39m\n\u001b[32m 22\u001b[39m \n\u001b[32m 23\u001b[39m \u001b[38;5;66;03m#all_prices = [p.text if p != None else \"_\" for p in all_prices]\u001b[39;00m\n\u001b[32m 24\u001b[39m writer.writerow([id, price_eur, price_cn, price_usd, bestprice_eur, bestprice_cn, bestprice_usd])\n\u001b[32m 25\u001b[39m time.sleep((random.randint(\u001b[32m2\u001b[39m, \u001b[32m3\u001b[39m))/\u001b[32m10\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m26\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m Exception \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 27\u001b[39m print(e)\n",
|
||
"\u001b[31mKeyboardInterrupt\u001b[39m: "
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"with open(\"./prices2.csv\", mode=\"a+\", encoding=\"utf8\", newline=\"\") as pricefile:\n",
|
||
" for idx, id in enumerate(get_all_ids()[3663:]):\n",
|
||
" try:\n",
|
||
" small_id = id.lower()\n",
|
||
"\n",
|
||
" response = rq.get(f\"https://www.merlinssteine.de/sets/{small_id}\")\n",
|
||
" soup = bs4.BeautifulSoup(response.text)\n",
|
||
"\n",
|
||
" # Prices\n",
|
||
" price_eur = soup.find(id=\"listprice_eur\")\n",
|
||
" price_usd = soup.find(id=\"listprice_usd\")\n",
|
||
" price_cn = soup.find(id=\"listprice_cn\")\n",
|
||
" bestprice_eur = soup.find(id=\"bestprice_eur\")\n",
|
||
" bestprice_usd = soup.find(id=\"bestprice_usd\")\n",
|
||
" bestprice_cn = soup.find(id=\"bestprice_cn\")\n",
|
||
"\n",
|
||
" #all_prices = [price_eur, price_cn, price_usd, bestprice_eur, bestprice_cn, bestprice_usd]\n",
|
||
" \n",
|
||
" #categories\n",
|
||
" #other_dump = [description.text.replace(\"\\n\", \"\") for description in soup.find_all(class_=\"setpage_ct\")]\n",
|
||
" writer = csv.writer(pricefile)\n",
|
||
" \n",
|
||
" #all_prices = [p.text if p != None else \"_\" for p in all_prices]\n",
|
||
" writer.writerow([id, price_eur, price_cn, price_usd, bestprice_eur, bestprice_cn, bestprice_usd])\n",
|
||
" time.sleep((random.randint(2, 3))/10)\n",
|
||
" except Exception as e:\n",
|
||
" print(e)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "4a10a1e3",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def split_by_keywords(text, keywords):\n",
|
||
" pattern = r'(' + '|'.join(map(re.escape, keywords)) + r')'\n",
|
||
" parts = re.split(pattern, text)\n",
|
||
" \n",
|
||
" result = {}\n",
|
||
" for i in range(1, len(parts), 2):\n",
|
||
" key = parts[i]\n",
|
||
" value = parts[i + 1].strip()\n",
|
||
" result[key] = value\n",
|
||
" \n",
|
||
" return result"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9c00f188",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"keywords = [\n",
|
||
" \"Listenpreis:\",\n",
|
||
" \"DetailsVon:\",\n",
|
||
" \"EAN:\",\n",
|
||
" \"Altersempfehlung:\",\n",
|
||
" \"Steine von:\",\n",
|
||
" \"Bestpreis:EU:\",\n",
|
||
" \"Bewertungen\",\n",
|
||
" \"Inhalt\",\n",
|
||
" \"PreiseListenpreis:\",\n",
|
||
" \"Hersteller-Kategorie:\",\n",
|
||
" \"Designer:\",\n",
|
||
" \"Maße:\",\n",
|
||
" \"Release:\",\n",
|
||
" \"Kategorien:\",\n",
|
||
" \"Hersteller-Videos\",\n",
|
||
" \"EU:\",\n",
|
||
" \"Anleitung\",\n",
|
||
" \"BewertungenCommunity:\",\n",
|
||
" \"Maßstab:\",\n",
|
||
" \"Erweiterung zu:\",\n",
|
||
" \"Reviews\",\n",
|
||
" \"Lizenz:\",\n",
|
||
" \"Farbverteilung\",\n",
|
||
" \"TeilelistenBrickLink\",\n",
|
||
" \"Bild:\",\n",
|
||
" \"ReviewsCommunity\",\n",
|
||
" \"Gewicht\",\n",
|
||
" \"Keine Aufkleber\",\n",
|
||
" \"Verpackungsmaße:\",\n",
|
||
" \"EU:Brickmo\",\n",
|
||
" \"Datenbanken:\",\n",
|
||
" \"Kategorie:\",\n",
|
||
" \"Keine Drucke\",\n",
|
||
" \"TechnikMOC:\",\n",
|
||
" \"Steingröße:\",\n",
|
||
" \"SonstigesMOC:\",\n",
|
||
" \"Variationen:\",\n",
|
||
" \"RebrickableVariation:\"\n",
|
||
"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "ae53869e",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'Listenpreis:': '14.95 EUR (7.3 ct/Teil) (7.9 ct/g)',\n",
|
||
" 'DetailsVon:': 'BlueBrixx',\n",
|
||
" 'EAN:': '4060904003671',\n",
|
||
" 'Steine von:': 'Qunlong',\n",
|
||
" 'Kategorie:': 'EisenbahnHersteller-',\n",
|
||
" 'Kategorien:': 'BBSpecial, BRIX',\n",
|
||
" 'Anleitung': 'Ohne Bauabschnitte',\n",
|
||
" 'Bewertungen': 'Bewerten',\n",
|
||
" 'Hersteller-Videos': 'video-1',\n",
|
||
" 'Inhalt': '205 Teile',\n",
|
||
" 'Gewicht': ': 190 g',\n",
|
||
" 'Keine Aufkleber': '',\n",
|
||
" 'Keine Drucke': '',\n",
|
||
" 'Farbverteilung': '',\n",
|
||
" 'TeilelistenBrickLink': 'XMLRebrickable CSVLEGO PaB CSVSetDB CSV',\n",
|
||
" 'PreiseListenpreis:': '14.95 EUR (7.3 ct/Teil) (7.9 ct/g)'}\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"details = {\n",
|
||
" \"id\" : [],\n",
|
||
" \"listprice_eur\" : [],\n",
|
||
" \"listprice_cn\" : [],\n",
|
||
" \"listprice_usd\" : [],\n",
|
||
" \"bestprice_eur\" : [],\n",
|
||
" \"bestprice_cn\" : [],\n",
|
||
" \"bestprice_usd\" : [],\n",
|
||
" \"brand\" : [],\n",
|
||
" \"ean\" : [],\n",
|
||
" \"producer\" : [],\n",
|
||
" \"release\" : [],\n",
|
||
" \"scale\" : [],\n",
|
||
" \"category\" : [],\n",
|
||
" \"producer_category\" : [],\n",
|
||
" \"num_parts\" : [],\n",
|
||
" \"width\" : [],\n",
|
||
" \"height\" : [],\n",
|
||
" \"depth\" : [],\n",
|
||
" \"designer\" : [],\n",
|
||
" \"weight\" : [],\n",
|
||
" \"age\" : []\n",
|
||
"}\n",
|
||
"import random\n",
|
||
"\n",
|
||
"me_details = pd.DataFrame(details)\n",
|
||
"\n",
|
||
"with open(\"../data/merlin/prices.csv\", mode=\"r\", encoding=\"utf8\") as price_file:\n",
|
||
" reader = csv.reader(price_file)\n",
|
||
"\n",
|
||
" # for row in reader:\n",
|
||
" # id, lp_eur, lp_cn, lp_usd, bp_eur, bp_cn, bp_usd, *other = row\n",
|
||
" \n",
|
||
" # me_details.loc[-1] = [id, lp_eur, lp_cn, lp_usd, bp_eur, bp_cn, bp_usd] + list(range(0, 12))\n",
|
||
" # me_details.index = me_details.index + 1\n",
|
||
" id, lp_eur, lp_cn, lp_usd, bp_eur, bp_cn, bp_usd, *other = [row for row in reader][random.randint(0, 4500)]\n",
|
||
" other = filter(lambda s: not \"Wikipedia\" in s, other)\n",
|
||
"\n",
|
||
" pprint.pp(split_by_keywords(\"\".join(other), keywords))\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b83aa413",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>id</th>\n",
|
||
" <th>listprice_eur</th>\n",
|
||
" <th>listprice_cn</th>\n",
|
||
" <th>listprice_usd</th>\n",
|
||
" <th>bestprice_eur</th>\n",
|
||
" <th>bestprice_cn</th>\n",
|
||
" <th>bestprice_usd</th>\n",
|
||
" <th>brand</th>\n",
|
||
" <th>ean</th>\n",
|
||
" <th>producer</th>\n",
|
||
" <th>...</th>\n",
|
||
" <th>scale</th>\n",
|
||
" <th>category</th>\n",
|
||
" <th>producer_category</th>\n",
|
||
" <th>num_parts</th>\n",
|
||
" <th>width</th>\n",
|
||
" <th>height</th>\n",
|
||
" <th>depth</th>\n",
|
||
" <th>designer</th>\n",
|
||
" <th>weight</th>\n",
|
||
" <th>age</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>0 rows × 21 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
"Empty DataFrame\n",
|
||
"Columns: [id, listprice_eur, listprice_cn, listprice_usd, bestprice_eur, bestprice_cn, bestprice_usd, brand, ean, producer, release, scale, category, producer_category, num_parts, width, height, depth, designer, weight, age]\n",
|
||
"Index: []\n",
|
||
"\n",
|
||
"[0 rows x 21 columns]"
|
||
]
|
||
},
|
||
"execution_count": 40,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"me_details"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "kgr",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.2"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|