first structure
commit
588e20b7b3
|
|
@ -0,0 +1,48 @@
|
|||
# HSMA Data Science and Analytics SS2024
|
||||
## ECG Data klassifier and segmentation
|
||||
|
||||
[This Project aims to klassify ... and semgment ... ECG Data]
|
||||
|
||||
## Table of Contents
|
||||
- [About](#about)
|
||||
- [Getting Started](#getting-started)
|
||||
- [Usage](#usage)
|
||||
- [Progress](#progress)
|
||||
- [Contributing](#contributing)
|
||||
- [License](#license)
|
||||
|
||||
## About
|
||||
[Provide a brief overview of the project, including its purpose and any relevant background information.]
|
||||
|
||||
## Getting Started
|
||||
[Instructions on how to get the project up and running on a local machine. Include prerequisites, installation steps, and any other necessary setup.]
|
||||
|
||||
### Prerequisites
|
||||
[List any software or tools that need to be installed before running the project.]
|
||||
|
||||
### Installation
|
||||
[Step-by-step guide on how to install the project.]
|
||||
|
||||
## Usage
|
||||
[Provide examples and instructions for using the project. Include any relevant code snippets or screenshots.]
|
||||
|
||||
## Progress
|
||||
- Data was searched and found at : https://doi.org/10.13026/wgex-er52
|
||||
- Data was cleaned:
|
||||
- Docker Container with MongoDB, because 10 GB and many arrays
|
||||
- Diagnosis from String to list
|
||||
- Data filtered to contain only healthy and the needed diagnosis data
|
||||
|
||||
|
||||
|
||||
## Contributing
|
||||
[Explain how others can contribute to the project. This might include guidelines for reporting bugs, submitting enhancements, or proposing new features.]
|
||||
|
||||
## License
|
||||
[Specify the license under which the project is distributed. Include any additional terms or conditions.]
|
||||
|
||||
## Acknowledgements
|
||||
[Optional section to thank individuals or organizations that have contributed to the project.]
|
||||
|
||||
## Contact
|
||||
[Provide contact information for inquiries or support.]
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
import wfdb
|
||||
import os
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
|
||||
# Directories and file paths
|
||||
# --------------------------------------------------------------------------------
|
||||
# Specify the directory where the WFDB records are stored
|
||||
project_dir = 'C:/Users/felix/OneDrive/Studium/Master MDS/1 Semester/DSA/physionet/large_12_ecg_data/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0'
|
||||
data_dir = project_dir + '/WFDBRecords'
|
||||
path_diag_lookup = "C:/Users/felix/OneDrive/Studium/Master MDS/1 Semester/DSA/physionet/large_12_ecg_data/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0/ConditionNames_SNOMED-CT.csv"
|
||||
#project_dir +'/ConditionNames_SNOMED-CT.csv'
|
||||
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
# print if project_dir exists
|
||||
if not os.path.exists("C:/Users/felix/OneDrive/Studium/Master MDS/1 Semester/DSA/physionet/large_12_ecg_data/a-large-scale-12-lead-electrocardiogram-database-for-arrhythmia-study-1.0.0/ConditionNames_SNOMED-CT.csv"):
|
||||
print(f"Directory {path_diag_lookup} does not exist")
|
||||
|
||||
|
||||
|
||||
def get_diagnosis_ids(record):
|
||||
# Get the diagnosis
|
||||
diagnosis = record.comments[2]
|
||||
# clean the diagnosis
|
||||
diagnosis = diagnosis.replace('Dx: ', '')
|
||||
list_diagnosis = [int(x.strip()) for x in diagnosis.split(',')]
|
||||
return list_diagnosis
|
||||
|
||||
def get_diagnosis_name(diagnosis):
|
||||
# get the diagnosis name from the lookup table
|
||||
name = [diagnosis_lookup[diagnosis_lookup['Snomed_CT'] == x]['Full Name'].to_string(index=False) for x in diagnosis]
|
||||
return name
|
||||
|
||||
def filter_signal_df_on_diag(df_dict, diagnosis_dict, filter_codes_df):
|
||||
# Create a list with filter codes and add 0 for padding
|
||||
filter_cod_li = list(filter_codes_df['Snomed_CT']) + [0]
|
||||
# Filter the diagnosis dictionary based on the filter codes
|
||||
filter_dict_diag = {k: v for k, v in diagnosis_dict.items() if all(i in filter_cod_li for i in v)}
|
||||
# Filter the df_dict based on the filtered_dict_diag
|
||||
filtered_df_dict = {i: df.loc[df.index.isin(filter_dict_diag.keys())] for i, df in df_dict.items()}
|
||||
|
||||
return filtered_df_dict
|
||||
|
||||
# --------------------------------------------------------------------------------
|
||||
# Explore the data
|
||||
# --------------------------------------------------------------------------------
|
||||
# Read the diagnosis lookup table
|
||||
diagnosis_lookup = pd.read_csv(path_diag_lookup)
|
||||
#print(diagnosis_lookup.head())
|
||||
|
||||
# Filter data based on the diagnosis
|
||||
|
||||
|
||||
|
||||
# ----------------------------------------------
|
||||
healthy_codes = [426177001, 426783006]
|
||||
|
||||
|
||||
categories = {
|
||||
'Gesund': [426177001, 426783006], # '426177001', '426783006
|
||||
'Herzrhythmusstörungen': [164890007, 427084000, 164889003, 426761007, 713422000, 427393009, 284470004, 17338001],
|
||||
'Leitungsstörungen': [270492004, 233917008, 59118001, 164909002, 698252002],
|
||||
'EKG-Welle': [164934002, 59931005, 428750005, 164917005, 429622005, 164930006, 164931005, 164912004, 164937009],
|
||||
'Spannungsänderungen': [39732003, 47665007, 251146004, 251199005],
|
||||
'Hypertrophien': [164873001, 89792004],
|
||||
'QT': [111975006],
|
||||
'Repolarisation': [428417006],
|
||||
'Myokardinfarkt': [164865005]
|
||||
}
|
||||
|
||||
|
||||
|
||||
diag_dict = {k: 0 for k in categories.keys()}
|
||||
|
||||
# Create a counter for the number of records
|
||||
counter = 0
|
||||
max_counter = 100_000#100_000
|
||||
|
||||
# Loop through the records
|
||||
for dir_th in os.listdir(data_dir):
|
||||
path_to_1000_records = data_dir + '/' + dir_th
|
||||
for dir_hd in os.listdir(path_to_1000_records):
|
||||
path_to_100_records = path_to_1000_records + '/' + dir_hd
|
||||
for record_name in os.listdir(path_to_100_records):
|
||||
# check if .hea is in the record_name
|
||||
if '.hea' not in record_name:
|
||||
continue
|
||||
# Remove the .hea extension from record_name
|
||||
record_name = record_name.replace('.hea', '')
|
||||
try:
|
||||
# Read the record
|
||||
record = wfdb.rdrecord(path_to_100_records + '/' + record_name)
|
||||
# Get the diagnosis
|
||||
diagnosis = np.array(get_diagnosis_ids(record))
|
||||
|
||||
# check if diagnosis is a subset of one of the categories
|
||||
for category_name, category_codes in categories.items():
|
||||
if set(diagnosis).issubset(set(category_codes)):
|
||||
# Increment the counter for the category
|
||||
diag_dict[category_name] += 1
|
||||
break
|
||||
|
||||
# Increment the counter
|
||||
counter += 1
|
||||
counter_bool = counter >= max_counter
|
||||
# Break the loop if we have read max_counter records
|
||||
if counter % 100 == 0:
|
||||
print(f"Read {counter} records")
|
||||
|
||||
if counter_bool:
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Failed to read record {record_name} due to ValueError")
|
||||
if counter_bool:
|
||||
break
|
||||
if counter_bool:
|
||||
break
|
||||
|
||||
"""
|
||||
ID: Herzrhythmusstörungen, Count: 22571
|
||||
ID: Leitungsstörungen, Count: 505
|
||||
ID: EKG-Welle, Count: 2067
|
||||
ID: Spannungsänderungen, Count: 613
|
||||
ID: Hypertrophien, Count: 5
|
||||
ID: QT, Count: 43
|
||||
ID: Repolarisation, Count: 73
|
||||
ID: Myokardinfarkt, Count: 1
|
||||
|
||||
"""
|
||||
|
||||
# # get the data
|
||||
# dict_healthy, dict_afib, dict_mi = get_diag_filtered_data_dict()
|
||||
|
||||
# # get unique diagnosis codes
|
||||
# unique_health_codes = np.unique(np.array([np.array(get_diagnosis_ids(d)) for d in dict_healthy.values()]).flatten())
|
||||
# unique_afib_codes = np.unique(np.array([np.array(get_diagnosis_ids(d)) for d in dict_afib.values()]).flatten())
|
||||
# unique_mi_codes = np.unique(np.array([np.array(get_diagnosis_ids(d)) for d in dict_mi.values()]).flatten())
|
||||
|
||||
# print(unique_health_codes)
|
||||
# print(unique_afib_codes)
|
||||
# print(unique_mi_codes)
|
||||
|
||||
# print(dict_healthy['JS00004'].__dict__)
|
||||
|
||||
|
||||
#print(diag_dict)
|
||||
for id, count in diag_dict.items():
|
||||
print(f"ID: {id}, Count: {count}")
|
||||
|
||||
print(f'Number of counter diagnoses: {len(diag_dict)}')
|
||||
print(f'Number of diagnoses in the lookup table: {len(diagnosis_lookup)}')
|
||||
print('found in the lookup table: ', len(diag_dict) == len(diagnosis_lookup))
|
||||
|
||||
# flatten the counters and count the unique values
|
||||
# healthy_counter = np.array(healthy_counter).flatten()
|
||||
# afib_counter = np.array(afib_counter).flatten()
|
||||
# mi_counter = np.array(mi_counter).flatten()
|
||||
|
||||
# unique_health_codes, counts_health = np.unique(healthy_counter, return_counts=True)
|
||||
# unique_afib_codes, counts_afib = np.unique(afib_counter, return_counts=True)
|
||||
# unique_mi_codes, counts_mi = np.unique(mi_counter, return_counts=True)
|
||||
|
||||
# print(unique_health_codes)
|
||||
# print(counts_health)
|
||||
# print(unique_afib_codes)
|
||||
# print(counts_afib)
|
||||
# print(unique_mi_codes)
|
||||
# print(counts_mi)
|
||||
|
||||
# # get the names of the diagnosis
|
||||
# names_health = get_diagnosis_name(unique_health_codes)
|
||||
# names_afib = get_diagnosis_name(unique_afib_codes)
|
||||
# names_mi = get_diagnosis_name(unique_mi_codes)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue