1826514
/
DSA_SS24
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of https://gitty.informatik.hs-mannheim.de/1826514/DSA_SS24

main
klara 2024-06-05 12:54:52 +02:00
parent ee5e2fbb6d a788038217
commit 5eb1c44403
2 changed files with 62 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -2,8 +2,12 @@
This project was developed through the Data Science and Analytics course at the Mannheim University of Applied Sciences. A data science cycle was taught theoretically on the basis of lectures and implemented practically in the project.
<<<<<<< HEAD
# Analysis of cardiovascular diseases using ECG data
=======
## Analysis of cardiovascular diseases using ECG data
>>>>>>> bc4897dacb83ed2c3722b7b5d66b2fff23fdcb6a
## Table of Contents

83
scripts/generate_data.py
View File

@ -1,49 +1,35 @@
"""
This script reads the WFDB records and extracts the diagnosis information from the comments.
The diagnosis information is then used to classify the records into categories.
The categories are defined by the diagnosis codes in the comments.
The records are then saved to pickle files based on the categories.
"""
import wfdb
import os
import numpy as np
import pickle
import bz2
import numpy as np
import pandas as pd
# Directories and file paths
# --------------------------------------------------------------------------------
# NOTE: Specify the directory where the WFDB records are stored
project_dir = 'C:/Users/felix/OneDrive/Studium/Master MDS/1 Semester/DSA/physionet/large_12_ecg_data/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0'
data_dir = project_dir + '/WFDBRecords'
path_diag_lookup = project_dir + "/ConditionNames_SNOMED-CT.csv"
# --------------------------------------------------------------------------------
# Functions
# Funktionen zum Bearbeiten der Daten
def get_diagnosis_ids(record):
"""
Extracts diagnosis IDs from a record and returns them as a list.
Args:
record (object): The record object containing the diagnosis information.
Returns:
list: A list of diagnosis IDs extracted from the record.
"""
# Get the diagnosis
diagnosis = record.comments[2]
# clean the diagnosis
diagnosis = diagnosis.replace('Dx: ', '')
list_diagnosis = [int(x.strip()) for x in diagnosis.split(',')]
return list_diagnosis
# --------------------------------------------------------------------------------
# Generate the data
# --------------------------------------------------------------------------------
if __name__ == '__main__':
"""
The following categories are used to classify the records:
SB, Sinusbradykardie
AFIB, Vorhofflimmern und Vorhofflattern (AFL)
GSVT, supraventrikulärer Tachykardie, Vorhoftachykardie, AV-Knoten-Reentry-Tachykardie, AV-Reentry-Tachykardie, Vorhofschrittmacher
SR Sinusrhythmus und Sinusunregelmäßigkeiten
"""
def get_diagnosis_name(diagnosis):
name = [diagnosis_lookup[diagnosis_lookup['Snomed_CT'] == x]['Full Name'].to_string(index=False) for x in diagnosis]
return name
def filter_signal_df_on_diag(df_dict, diagnosis_dict, filter_codes_df):
filter_cod_li = list(filter_codes_df['Snomed_CT']) + [0]
filter_dict_diag = {k: v for k, v in diagnosis_dict.items() if all(i in filter_cod_li for i in v)}
filtered_df_dict = {i: df.loc[df.index.isin(filter_dict_diag.keys())] for i, df in df_dict.items()}
return filtered_df_dict
# Verzeichnisse und Dateipfade
project_dir = 'C:/Users/arman/PycharmProjects/pythonProject/DSA/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0'
data_dir = project_dir + '/WFDBRecords'
path_diag_lookup = "C:/Users/arman/PycharmProjects/pythonProject/DSA/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0/ConditionNames_SNOMED-CT.csv"
# Daten erkunden
diagnosis_lookup = pd.read_csv(path_diag_lookup)
categories = {
'SB': [426177001],
'AFIB': [164889003, 164890007],
@ -52,54 +38,41 @@ if __name__ == '__main__':
}
diag_dict = {k: [] for k in categories.keys()}
# Create a counter for the number of records
counter = 0
max_counter = 100_000
failed_records = []
# Loop through the records
for dir_th in os.listdir(data_dir):
path_to_1000_records = data_dir + '/' + dir_th
for dir_hd in os.listdir(path_to_1000_records):
path_to_100_records = path_to_1000_records + '/' + dir_hd
for record_name in os.listdir(path_to_100_records):
# check if .hea is in the record_name
if '.hea' not in record_name:
continue
# Remove the .hea extension from record_name
record_name = record_name.replace('.hea', '')
try:
# Read the record
record = wfdb.rdrecord(path_to_100_records + '/' + record_name)
# Get the diagnosis
diagnosis = np.array(get_diagnosis_ids(record))
# check if diagnosis is a subset of one of the categories
for category_name, category_codes in categories.items():
# if any of the diagnosis codes is in the category_codes
if any(i in category_codes for i in diagnosis):
diag_dict[category_name].append(record)
break
# Increment the counter of how many records we have read
counter += 1
counter_bool = counter >= max_counter
# Break the loop if we have read max_counter records
if counter % 100 == 0:
print(f"Read {counter} records")
print(f"Gelesen {counter} Datensätze")
if counter_bool:
break
except Exception as e:
failed_records.append(record_name)
print(f"Failed to read record {record_name} due to ValueError. Sum of failed records: {len(failed_records)}")
print(f"Fehler beim Lesen des Datensatzes {record_name}: {e}")
if counter_bool:
break
if counter_bool:
break
# write to pickle
for cat_name, records in diag_dict.items():
print(f"Writing {cat_name} to pickle with {len(records)} records")
# if path not exists create it
print(f"Schreibe {cat_name} in eine komprimierte Datei mit {len(records)} Datensätzen")
if not os.path.exists('./data'):
os.makedirs('./data')
with open(f'./data/{cat_name}.pkl', 'wb') as f:
compressed_filename = f'./data/{cat_name}.pkl.bz2'
with bz2.open(compressed_filename, 'wb') as f:
pickle.dump(records, f)