extended data quality checks
parent
750efd04fc
commit
6b383a5276
|
|
@ -55,26 +55,119 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"execution_count": 18,
|
"execution_count": 12,
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"name": "stdout",
|
"name": "stdout",
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"Missing timeseries in 0 records\n"
|
"First record in SB: ['Age: 59', 'Sex: Female', 'Dx: 426177001,164934002', 'Rx: Unknown', 'Hx: Unknown', 'Sx: Unknown']\n",
|
||||||
|
"Missing sex in JS34080 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS12543 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS12571 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS12576 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS12576 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS12609 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS12609 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS13024 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS13024 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS13504 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13505 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13575 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13583 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13645 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13646 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS13647 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS14027 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS14050 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS14498 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS14555 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS14995 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS14995 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS18505 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18506 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18507 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18508 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18509 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18510 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18511 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18512 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18513 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18514 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS18515 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS18515 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS18574 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS19386 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS19447 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS10867 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS10867 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS11507 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS11507 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS22918 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS22918 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS23063 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS23063 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS23064 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS23787 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS23787 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS24143 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS24143 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS24144 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS24144 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS24145 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45355 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45356 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45357 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45358 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45359 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45360 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45360 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45361 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45361 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45364 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45367 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45367 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45369 and comments: Age: NaN\n",
|
||||||
|
"Missing age in JS45370 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45370 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45382 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45382 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45383 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45383 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45384 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45384 and comments: Sex: Unknown\n",
|
||||||
|
"Missing age in JS45385 and comments: Age: NaN\n",
|
||||||
|
"Missing sex in JS45385 and comments: Sex: Unknown\n",
|
||||||
|
"Missing timeseries in 0 records\n",
|
||||||
|
"Missing age in 55 records\n",
|
||||||
|
"Missing sex in 21 records\n"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
|
"# print first record and first record keys\n",
|
||||||
|
"print(f\"First record in SB: {data['SB'][0].comments}\")\n",
|
||||||
|
"\n",
|
||||||
"missing_timeseries = []\n",
|
"missing_timeseries = []\n",
|
||||||
|
"missing_age = []\n",
|
||||||
|
"missing_sex = []\n",
|
||||||
"for cat_name, records in data.items():\n",
|
"for cat_name, records in data.items():\n",
|
||||||
" for record in records:\n",
|
" for record in records:\n",
|
||||||
" if len(record.p_signal) != 5000:\n",
|
" if len(record.p_signal) != 5000:\n",
|
||||||
" missing_timeseries.append(record)\n",
|
" missing_timeseries.append(record)\n",
|
||||||
" print(f\"Missing timeseries in {record.record_name}\")\n",
|
" print(f\"Missing timeseries in {record.record_name}\")\n",
|
||||||
" #if record.comments[2]== '':\n",
|
" #if record.comments[2]== '':\n",
|
||||||
"print(f\"Missing timeseries in {len(missing_timeseries)} records\")"
|
" if 'Age: ' not in record.comments[0] or record.comments[0] == 'Age: NaN':\n",
|
||||||
|
" missing_age.append(record)\n",
|
||||||
|
" print(f\"Missing age in {record.record_name} and comments: {record.comments[0]}\")\n",
|
||||||
|
" if record.comments[1] == 'Sex: Unknown' or record.comments[1] == '':\n",
|
||||||
|
" missing_sex.append(record)\n",
|
||||||
|
" print(f\"Missing sex in {record.record_name} and comments: {record.comments[1]}\")\n",
|
||||||
|
" \n",
|
||||||
|
"print(f\"Missing timeseries in {len(missing_timeseries)} records\")\n",
|
||||||
|
"print(f\"Missing age in {len(missing_age)} records\")\n",
|
||||||
|
"print(f\"Missing sex in {len(missing_sex)} records\")"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -253,7 +253,7 @@
|
||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.11.3"
|
"version": "3.10.4"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue