{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Naive Bayes examined 100 samples\n",
"---\n",
"Confusion Matrix\n",
"{'Iris-versicolor': 16, 'Iris-virginica': 3, 'Iris-setosa': 0}\n",
"{'Iris-versicolor': 1, 'Iris-virginica': 15, 'Iris-setosa': 0}\n",
"{'Iris-versicolor': 0, 'Iris-virginica': 0, 'Iris-setosa': 15}\n",
"\n",
"Accuracy: 92.0%\n",
"skLearn accuracy: 96.0%\n"
]
}
],
"source": [
"import math\n",
"import pandas as pd\n",
"from sklearn import preprocessing, tree, model_selection\n",
"from sklearn.naive_bayes import GaussianNB\n",
"from sklearn.datasets import load_iris\n",
"\n",
"filename = 'iris.csv'\n",
"needs_discretized = True\n",
"class_attr = 'class'\n",
"split = .67\n",
"classifier = 3\n",
"\n",
"def main():\n",
" # Read CSV\n",
" df = pd.read_csv(filename)\n",
" \n",
" # Randomize Order\n",
" df = df.sample(frac=1)\n",
"\n",
" # Discretize\n",
" if needs_discretized:\n",
" for col in df:\n",
" if col != class_attr:\n",
" df[col] = pd.qcut(df[col], q=5)\n",
" \n",
" # Split Data\n",
" if split != 1:\n",
" testing = df.head(-math.floor(len(df)*split))\n",
" data = df.head(math.floor(len(df)*split))\n",
" else:\n",
" testing = data = df\n",
" \n",
" # Choose Classifier\n",
" if classifier == 1:\n",
" r1(data, testing)\n",
" elif classifier == 2:\n",
" decision_tree(data, testing)\n",
" else:\n",
" naive_bayes(data, testing)\n",
" \n",
"def r1(data, testing):\n",
" # Set up big dictionary\n",
" rules = dict()\n",
" \n",
" for attr in data:\n",
" if attr != class_attr:\n",
" rules[attr] = dict()\n",
"\n",
" # Loop thru data\n",
" for attr in data:\n",
" if attr != class_attr:\n",
" freq = {v:{c:0 for c in data[class_attr].unique()} for v in data[attr].unique()}\n",
" for i, sample in data.iterrows():\n",
" freq[sample[attr]][sample[class_attr]] += 1\n",
" \n",
" attr_rule = dict()\n",
" error = 0\n",
" for (k,v) in freq.items():\n",
" rule = max(v, key=v.get)\n",
" for c in v:\n",
" if c != rule:\n",
" error += v[c]\n",
" attr_rule[k] = rule\n",
" error /= len(data)\n",
" rules[attr] = (attr_rule, error)\n",
" \n",
" # Select best attr\n",
" best_attr = min(rules, key=lambda x: rules[x][1])\n",
" rule = rules[best_attr][0]\n",
" print(f'R1 chose {best_attr}')\n",
" print(print_tree(rule))\n",
" print('---')\n",
" \n",
" confusion = {v:{c:0 for c in data[class_attr].unique()} for v in data[class_attr].unique()}\n",
" \n",
" correct = 0\n",
" for i, row in testing.iterrows():\n",
" confusion[row[class_attr]][rule[row[best_attr]]] += 1\n",
" if row[class_attr] == rule[row[best_attr]]: correct += 1\n",
" \n",
" print(\"Confusion Matrix\")\n",
" \n",
" for (actual,guess) in confusion.items():\n",
" print(guess)\n",
" print()\n",
" print(f'Accuracy: {round((correct/len(testing))*100, 3)}%')\n",
"\n",
"\n",
"def decision_tree(data, testing):\n",
" print(f'Decision Tree examined {len(data)} samples and built the following tree:', end='')\n",
" rules = recur_tree(data)\n",
" print_tree(rules)\n",
" print('\\n---')\n",
" print(\"Confusion Matrix\")\n",
" confusion, correct = {v:{c:0 for c in data[class_attr].unique()} for v in data[class_attr].unique()}, 0\n",
" \n",
" for i, row in testing.iterrows():\n",
" guess = test_tree(row, rules)\n",
" confusion[row[class_attr]][guess] += 1\n",
" if row[class_attr] == guess: correct += 1 \n",
" \n",
" for (actual,guess) in confusion.items():\n",
" print(guess)\n",
"\n",
" print()\n",
" print(f'Accuracy: {round((correct/len(testing))*100, 3)}%')\n",
" \n",
" # Test with sklearn tree\n",
" dtc = tree.DecisionTreeClassifier()\n",
" x,y = load_iris(return_X_y=True)\n",
" x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y, test_size=(1-split), random_state=0)\n",
" y_pred = dtc.fit(x_train, y_train).predict(x_test)\n",
" print(f'skLearn accuracy: {sum(y_pred == y_test)*100/len(y_pred)}%')\n",
"\n",
"def recur_tree(data):\n",
" rules = {}\n",
" \n",
" # Find info gain per attrT\n",
" info = calc_info(data)\n",
" if info == 0:\n",
" return data[class_attr].unique()[0]\n",
" \n",
" # gain = {attr:sum([info - calc_info(data[data[attr] == v]) for v in data[attr].unique()]) for attr in data if attr != class_attr}\n",
" gain = {attr:0 for attr in data if attr != class_attr}\n",
" for attr in gain:\n",
" for v in data[attr].unique():\n",
" gain[attr] += info - calc_info(data[data[attr] == v])\n",
" \n",
" # Choose highest info gain\n",
" attr = max(gain, key=gain.get)\n",
" if (gain[attr] == 0): \n",
" return data[class_attr].unique()[0]\n",
" \n",
" # Split data based on values of attr and recur\n",
" rules[attr] = {}\n",
" for v in data[attr].unique():\n",
" rules[attr][v] = recur_tree(data[data[attr] == v])\n",
"\n",
" return rules\n",
" \n",
"def calc_info(data):\n",
" return abs(sum([(count/len(data))*math.log((count/len(data)), 2) for count in data[class_attr].value_counts()]))\n",
" \n",
"def print_tree(rules, indent=0):\n",
" if type(rules) != dict: return rules\n",
" \n",
" for key in rules.keys():\n",
" print('\\n'+' '*3*indent + f'* {key}', end='')\n",
" s = print_tree(rules[key], indent + 1)\n",
" if s: print(f' --> {s}', end='')\n",
" \n",
" return None\n",
"\n",
"def test_tree(row, rules):\n",
" if type(rules) != dict: return rules\n",
" \n",
" attr = list(rules.keys())[0]\n",
" return test_tree(row, rules[attr][row[attr]])\n",
"\n",
"def naive_bayes(data, testing):\n",
" confusion, correct = {v:{c:0 for c in data[class_attr].unique()} for v in data[class_attr].unique()}, 0\n",
" class_freq = {c:(len(data[data[class_attr] == c])) for c in data[class_attr].unique()}\n",
" for i, row in testing.iterrows():\n",
" probs = {c:(len(data[data[class_attr] == c]))/len(data) for c in data[class_attr].unique()}\n",
" \n",
" for attr in data:\n",
" if attr != class_attr:\n",
" same_value = data[data[attr] == row[attr]]\n",
" for c in class_freq.keys():\n",
" probs[c] *= len(same_value[same_value[class_attr] == c])/class_freq[c]\n",
" \n",
" guess = max(probs, key=probs.get)\n",
" confusion[row[class_attr]][guess] += 1\n",
" if row[class_attr] == guess: correct += 1\n",
" \n",
" print(f'Naive Bayes examined {len(data)} samples')\n",
" print('---')\n",
" print(\"Confusion Matrix\")\n",
" for (actual,guess) in confusion.items():\n",
" print(guess)\n",
" print()\n",
" print(f'Accuracy: {round((correct/len(testing))*100, 3)}%')\n",
" \n",
" # Test with sklearn GaussianNaiveBayes\n",
" nb = GaussianNB()\n",
" x,y = load_iris(return_X_y=True)\n",
" x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y, test_size=(1-split), random_state=0)\n",
" y_pred = nb.fit(x_train, y_train).predict(x_test)\n",
" print(f'skLearn accuracy: {sum(y_pred == y_test)*100/len(y_pred)}%')\n",
" \n",
"if __name__ == '__main__':\n",
" main()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python (default)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}