{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p  style=\"z-index: 101;background: #fde073;text-align: center;line-height: 2.5;overflow: hidden;font-size:22px;\">Please <a href=\"https://www.pycm.io/doc/#Cite\" target=\"_blank\">cite us</a> if you use the software</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example-8 (Confidence interval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Environment check"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking that the notebook is running on Google Colab or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "try:\n",
    "  import google.colab\n",
    "  !{sys.executable} -m pip -q -q install pycm\n",
    "except:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hide_output": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyring is skipped due to an exception: invalid syntax (core.py, line 48)\n"
     ]
    }
   ],
   "source": [
    "!{sys.executable} -m pip -q -q install matplotlib;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pycm\n",
    "\n",
    "def plot_ci(cm, param, alpha=0.05, method=\"normal-approx\"):\n",
    "    \"\"\"\n",
    "    Plot two-sided confidence interval.\n",
    "    \n",
    "    :param cm: ConfusionMatrix\n",
    "    :type cm : pycm.ConfusionMatrix object\n",
    "    :param param: input parameter\n",
    "    :type param: str\n",
    "    :param alpha: type I error\n",
    "    :type alpha: float\n",
    "    :param method: binomial confidence intervals method\n",
    "    :type method: str\n",
    "    :return: None\n",
    "    \"\"\"\n",
    "    conf_str = str(round(100 * (1-alpha)))\n",
    "    print(conf_str + \"%CI :\")\n",
    "    if param in cm.class_stat.keys():\n",
    "        mean = []\n",
    "        error = [[], []]\n",
    "        data = cm.CI(param, alpha=alpha, binom_method=method)\n",
    "        class_names_str = list(map(str,(cm.classes)))\n",
    "        for class_index, class_name in enumerate(cm.classes):\n",
    "            print(str(class_name) +\" : \" + str(data[class_name][1]))\n",
    "            mean.append(cm.class_stat[param][class_name])\n",
    "            error[0].append(cm.class_stat[param][class_name] - data[class_name][1][0])\n",
    "            error[1].append(data[class_name][1][1] - cm.class_stat[param][class_name])\n",
    "        fig = plt.figure()\n",
    "        plt.errorbar(mean, class_names_str, xerr=error, fmt='o', capsize=5, linestyle=\"dotted\")\n",
    "        plt.ylabel('Class')\n",
    "        fig.suptitle(\"Param :\" + param + \", Alpha:\" + str(alpha), fontsize=16)\n",
    "        for index,value in enumerate(mean):\n",
    "            down_point = data[cm.classes[index]][1][0]\n",
    "            up_point = data[cm.classes[index]][1][1]\n",
    "            plt.text(value, class_names_str[index], \"%f\" % value, ha=\"center\", va=\"top\", color=\"red\")\n",
    "            plt.text(down_point, class_names_str[index], \"%f\" % down_point, ha=\"right\", va=\"bottom\", color=\"red\")\n",
    "            plt.text(up_point , class_names_str[index], \"%f\" % up_point, ha=\"left\", va=\"bottom\", color=\"red\")\n",
    "    else:\n",
    "        mean = cm.overall_stat[param]\n",
    "        data = cm.CI(param, alpha=alpha, binom_method=method)\n",
    "        print(data[1])\n",
    "        error = [[], []]\n",
    "        up_point = data[1][1]\n",
    "        down_point = data[1][0]\n",
    "        error[0] = [cm.overall_stat[param] - down_point]\n",
    "        error[1] = [up_point - cm.overall_stat[param]]\n",
    "        fig = plt.figure()\n",
    "        plt.errorbar(mean, [param], xerr=error, fmt='o',capsize=5, linestyle=\"dotted\")\n",
    "        fig.suptitle(\"Alpha:\" + str(alpha), fontsize=16)\n",
    "        plt.text(mean, param, \"%f\" % mean, ha=\"center\", va=\"top\", color=\"red\")\n",
    "        plt.text(down_point, param, \"%f\" % down_point, ha=\"right\", va=\"bottom\", color=\"red\")\n",
    "        plt.text(up_point, param, \"%f\" % up_point, ha=\"left\", va=\"bottom\", color=\"red\")\n",
    "    \n",
    "        \n",
    "    plt.show()\n",
    "            \n",
    "            \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = pycm.ConfusionMatrix(matrix={0:{0:13,1:2,2:5},1:{0:1,1:10,2:6},2:{0:2,1:0,2:9}})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.4409588557245249, 0.8590411442754751)\n",
      "1 : (0.3542806291426003, 0.8221899590926939)\n",
      "2 : (0.5902508705143656, 1.0461127658492708)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19064 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"TPR\",method=\"normal-approx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.43285051021946463, 0.8188104521320554)\n",
      "1 : (0.36005074385515545, 0.7838921396086862)\n",
      "2 : (0.5230138624217553, 0.9486333993289995)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19064 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"TPR\",method=\"wilson\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.4315849969359111, 0.8200759654156089)\n",
      "1 : (0.3595423027276775, 0.7844005807361641)\n",
      "2 : (0.5115131538244717, 0.9601341079262828)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 18840 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"TPR\",method=\"agresti-coull\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (-0.0074215352310388755, 0.22170724951675308)\n",
      "1 : (-0.021966229952848706, 0.15099848801736493)\n",
      "2 : (0.1500197639598834, 0.44457483063471115)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 18788 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"FPR\",method=\"normal-approx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.03711769205479948, 0.27196225367657073)\n",
      "1 : (0.017874407775063175, 0.20718988318746445)\n",
      "2 : (0.17489302346408198, 0.45783439172043583)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19104 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"FPR\",method=\"wilson\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.02898740902933511, 0.28009253670203504)\n",
      "1 : (0.0075968375750390255, 0.21746745338748863)\n",
      "2 : (0.1737338065288983, 0.4589936086556194)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19192 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"FPR\",method=\"agresti-coull\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (0.6399211771547619, 0.902935965702381)\n",
      "1 : (0.6273084086303518, 0.8964107564550372)\n",
      "2 : (0.6151497954659743, 0.9057347254185467)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 18080 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"AUC\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "0 : (1.986202987452899, 18.530051901514057)\n",
      "1 : (2.2523561191462638, 36.90867850896665)\n",
      "2 : (1.5589394210441203, 4.858346516200761)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19076 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"PLR\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall ACC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "(0.5333055584484714, 0.8000277748848619)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 8892 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"Overall ACC\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95%CI :\n",
      "(0.31072820940081924, 0.7046564059837961)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 8640 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_ci(cm,param=\"Kappa\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<script type=\"text/javascript\">\n",
    "var sc_project=12845182; \n",
    "var sc_invisible=1; \n",
    "var sc_security=\"001dde8f\"; \n",
    "</script>\n",
    "<script type=\"text/javascript\"\n",
    "src=\"https://www.statcounter.com/counter/counter.js\" async></script>\n",
    "<noscript><div class=\"statcounter\"><a title=\"Web Analytics\"\n",
    "href=\"https://statcounter.com/\" target=\"_blank\"><img class=\"statcounter\"\n",
    "src=\"https://c.statcounter.com/12845182/0/001dde8f/1/\" alt=\"Web Analytics\"\n",
    "referrerPolicy=\"no-referrer-when-downgrade\"></a></div></noscript>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
