181 lines
7.0 KiB
Plaintext
181 lines
7.0 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ee5d9084",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Bricklink"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 83,
|
|
"id": "6a0f9ac8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import csv\n",
|
|
"import requests as rq\n",
|
|
"from bs4 import BeautifulSoup\n",
|
|
"from bs4.element import NavigableString"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c31288a3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def process_page(page : int):\n",
|
|
" with open(f\"./data/bricklink/parts_{page}.html\", mode=\"r\", encoding=\"utf8\") as pagefile:\n",
|
|
" s = BeautifulSoup(pagefile.read().replace(\"\\n\", \"\"))\n",
|
|
"\n",
|
|
" form = s.find(id=\"ItemEditForm\")\n",
|
|
" table = form.find_all(name=\"table\", class_=\"bg-color--white catalog-list__body-main catalog-list__body-main--alternate-row\")[0].find(name=\"tbody\")\n",
|
|
" with open(\"data/bricklink/parts.csv\", mode=\"a+\", encoding=\"utf8\", newline='') as parts_file:\n",
|
|
" writer = csv.writer(parts_file)\n",
|
|
" writer.writerow([\"part_id\", \"part_name\", \"categories\"])\n",
|
|
" for row in list(table.children)[1:]:\n",
|
|
" try:\n",
|
|
" img, nr, description = list(row.children)\n",
|
|
" \n",
|
|
" part_id = nr.find(name=\"a\").text\n",
|
|
" part_name = description.find(name=\"strong\").text\n",
|
|
"\n",
|
|
" links = description.find_all(name=\"a\")\n",
|
|
"\n",
|
|
" categories = \", \".join([link.text for link in links if link.text != \"Catalog\" and link.text != \"Parts\"])\n",
|
|
"\n",
|
|
" writer.writerow([part_id, part_name, categories])\n",
|
|
" except Exception as e:\n",
|
|
" print(str(row), e)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 63,
|
|
"id": "9b58d65d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def process_page_minifigs(page : int):\n",
|
|
" with open(f\"./data/bricklink/minifigs_{page}.html\", mode=\"r\", encoding=\"utf8\") as pagefile:\n",
|
|
" s = BeautifulSoup(pagefile.read().replace(\"\\n\", \"\"))\n",
|
|
"\n",
|
|
" form = s.find(id=\"ItemEditForm\")\n",
|
|
" table = form.find_all(name=\"table\", class_=\"bg-color--white catalog-list__body-main catalog-list__body-main--alternate-row\")[0].find(name=\"tbody\")\n",
|
|
" with open(\"data/bricklink/minifigs.csv\", mode=\"a+\", encoding=\"utf8\", newline='') as parts_file:\n",
|
|
" writer = csv.writer(parts_file)\n",
|
|
" writer.writerow([\"minifig_id\", \"minifig_name\", \"categories\"])\n",
|
|
" for row in list(table.children)[1:]:\n",
|
|
" try:\n",
|
|
" img, nr, description = list(row.children)\n",
|
|
" \n",
|
|
" minifig_id = nr.find(name=\"a\").text\n",
|
|
" minifig_name = description.find(name=\"strong\").text\n",
|
|
"\n",
|
|
" links = description.find_all(name=\"a\")\n",
|
|
"\n",
|
|
" categories = \", \".join([link.text for link in links if link.text != \"Catalog\" and link.text != \"Parts\"])\n",
|
|
"\n",
|
|
" writer.writerow([minifig_id, minifig_name, categories])\n",
|
|
" except Exception as e:\n",
|
|
" print(str(row), e)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 98,
|
|
"id": "0fc0394d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def process_page_sets(page : int):\n",
|
|
" with open(f\"./data/bricklink/sets_{page}.html\", mode=\"r\", encoding=\"utf8\") as pagefile:\n",
|
|
" s = BeautifulSoup(pagefile.read().replace(\"\\n\", \"\"))\n",
|
|
"\n",
|
|
" form = s.find(id=\"ItemEditForm\")\n",
|
|
" table = form.find_all(name=\"table\", class_=\"bg-color--white catalog-list__body-main catalog-list__body-main--alternate-row\")[0].find(name=\"tbody\")\n",
|
|
" with open(\"data/bricklink/sets.csv\", mode=\"a+\", encoding=\"utf8\", newline='') as parts_file:\n",
|
|
" writer = csv.writer(parts_file)\n",
|
|
" writer.writerow([\"set_id\", \"set_name\", \"categories\", \"parts\", \"minifigs\", \"year\"])\n",
|
|
" for row in list(table.children)[1:]:\n",
|
|
" try:\n",
|
|
" img, nr, description = list(row.children)\n",
|
|
" \n",
|
|
" set_id = nr.find(name=\"a\").text\n",
|
|
" set_name = description.find(name=\"strong\").text\n",
|
|
"\n",
|
|
" links = description.find_all(name=\"a\")\n",
|
|
"\n",
|
|
" font = description.find(name=\"font\", class_=\"fv\")\n",
|
|
" infos = \"\".join(child for child in font.contents if isinstance(child, NavigableString)).strip(\"\\n\").split(\",\")\n",
|
|
"\n",
|
|
" extracted = {\n",
|
|
" \"Part\" : \"\",\n",
|
|
" \"Minifigure\" : \"\",\n",
|
|
" \"Year\" : \"\"\n",
|
|
" }\n",
|
|
"\n",
|
|
" for info in infos:\n",
|
|
" if \"Part\" in info:\n",
|
|
" extracted[\"Part\"] = info.split(\" \")[0]\n",
|
|
" if \"Minifigure\" in info:\n",
|
|
" extracted[\"Minifigure\"] = info.split(\" \")[0]\n",
|
|
" else:\n",
|
|
" extracted[\"Year\"] = info.split(\":\")[0].strip()\n",
|
|
"\n",
|
|
" categories = \", \".join([link.text for link in links if link.text != \"Catalog\" and link.text != \"Sets\"])\n",
|
|
"\n",
|
|
" writer.writerow([set_id, set_name, categories, extracted[\"Part\"], extracted[\"Minifigure\"], extracted[\"Year\"]])\n",
|
|
" except Exception as e:\n",
|
|
" print(str(row), e)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 99,
|
|
"id": "3fa694ef",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" 'NavigableString' object has no attribute 'children'\n",
|
|
" 'NavigableString' object has no attribute 'children'\n",
|
|
" 'NavigableString' object has no attribute 'children'\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for i in [1,2,3]:\n",
|
|
" process_page_sets(i)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "venv (3.14.4)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.14.4"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|