빅데이터 입문 기말 리포트

import csv
import pickle
with open("./NHIS_OPEN_GJ_2015.csv") as csvfile:
    reader = csv.reader(csvfile)
    header = next(reader, None)
    #header =  {header[i]:i  for i in range(0, len(header))}
    table = list()
    for row in reader:
        row = {header[i] : row[i] for i in range(len(header))}
        if row["신장(5Cm단위)"] == "" or row["체중(5Kg 단위)"] == "" or row["허리둘레"] == "": 
            continue

        bmi =  round(10000 * int(row["신장(5Cm단위)"]) / (int(row["체중(5Kg 단위)"]) ** 2), 2)
        label = 1
        if bmi > 40:label = 4
        if bmi > 35:label = 3
        if bmi > 30:label = 2

        nrow = {
            "height":int(row["신장(5Cm단위)"]),
            "weight":int(row["체중(5Kg 단위)"]),
            "waist":int(row["허리둘레"]),
            "overweight":label
            }
        
        table.append(nrow)
    with open("./preprocessing2015.pkl","wb+") as fw:
        pickle.dump(table, fw, protocol = pickle.HIGHEST_PROTOCOL)

SVM

from sklearn.svm import SVC
import pickle
clf= SVC(kernel='poly', probability=True)
x = pickle.load(open('preprocessing2016.pkl', "rb"))
y = pickle.load(open('preprocessing2015.pkl', "rb"))
print(len(x))
print(len(y))
data_test = [[row["weight"], row["height"]] for row in y]
labels_test = [row["overweight"] for row in y]
data_train = [[row["weight"], row["height"]] for row in x]
labels_train = [row["overweight"] for row in x]
## 3-1) training
clf.fit(data_train, labels_train)
## 3-2) prediction
prediction_test= clf.predict(data_test)
probability_test= clf.predict_proba(data_test)[:,1]
## 4. Evaluate the model
fromsklearn.metricsimportconfusion_matrix, roc_auc_score
## 4-1) AUC
auc= roc_auc_score(labels_test, prob_test)
## 4-2) Confusion matrix
tn, fp, fn, tp= confusion_matrix(labels_test, pred_test).ravel()
accuracy = (tp+ tn) / (tn+ fp+ fn+ tp)
sensitivity = tp/ (tp+ fn)
specificity = tn/ (fp+ tn)
precision = tp/ (tp+ fp)
recall = tp/ (tp+ fn)
fscore= 2 * precision * recall / (precision + recall)