testing-exercises.ipynb · kaibo/Data-Analysis

加入 Gitee

与超过 1200万开发者一起发现、参与优秀开源项目，私有仓库也完全免费：）

文件

克隆/下载

testing-exercises.ipynb 15.64 KB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:38:38.465675Z",
     "start_time": "2019-02-23T16:38:38.422180Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script><script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window._Plotly) {require(['plotly'],function(plotly) {window._Plotly=plotly;});}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../..')\n",
    "\n",
    "# Options for pandas\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_rows = 10\n",
    "\n",
    "# Display all cell outputs\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "from plotly.offline import iplot, init_notebook_mode\n",
    "init_notebook_mode(connected=True)\n",
    "\n",
    "import cufflinks\n",
    "cf.go_offline(connected=True)\n",
    "cf.set_config_file(theme='pearl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:39:00.607978Z",
     "start_time": "2019-02-23T16:39:00.567876Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 100)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(1000, 100))\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:39:16.845392Z",
     "start_time": "2019-02-23T16:39:16.774748Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 100)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs = df.corr()\n",
    "corrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:42:26.032480Z",
     "start_time": "2019-02-23T16:42:25.998079Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 3,  8, 44, 45, 54, 96]), array([54, 96, 45, 44,  3,  8]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = -0.1\n",
    "direction = 'less'\n",
    "\n",
    "if direction == 'greater':\n",
    "    values_index = np.where(corrs > threshold)\n",
    "elif direction == 'less':\n",
    "    values_index = np.where(corrs < threshold)\n",
    "    \n",
    "values_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:47:56.940313Z",
     "start_time": "2019-02-23T16:47:56.909882Z"
    }
   },
   "outputs": [],
   "source": [
    "rows_index = values_index[0]\n",
    "columns_index = values_index[1]\n",
    "\n",
    "pairs = list(map(tuple, set([frozenset((x, y)) for x, y in zip(rows_index, columns_index)])))\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "# Counter(pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:47:57.429941Z",
     "start_time": "2019-02-23T16:47:57.397928Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 54), (8, 96), (44, 45)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T17:04:56.074717Z",
     "start_time": "2019-02-23T17:04:56.041811Z"
    }
   },
   "outputs": [],
   "source": [
    "subset_df = pd.DataFrame(dict(value=corrs.values[values_index], var1=corrs.index[values_index[0]],\n",
    "                         var2=corrs.columns[values_index[1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-24T20:36:52.221603Z",
     "start_time": "2019-02-24T20:36:52.182531Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.111172</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.117402</td>\n",
       "      <td>8</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.104640</td>\n",
       "      <td>44</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      value  var1  var2\n",
       "0 -0.111172     3    54\n",
       "1 -0.117402     8    96\n",
       "2 -0.104640    44    45"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_df.iloc[:int(len(subset_df)/2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T17:05:53.369263Z",
     "start_time": "2019-02-23T17:05:53.337720Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "var1  var2\n",
       "3     54      1\n",
       "8     96      1\n",
       "44    45      1\n",
       "45    44      1\n",
       "54    3       1\n",
       "96    8       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_df.groupby(['var1', 'var2']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-24T14:57:30.059725Z",
     "start_time": "2019-02-24T14:57:30.027029Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['value', 'variable1', 'var2'], dtype='object')"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_df.columns.str.replace('var1', 'variable1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:50:24.548345Z",
     "start_time": "2019-02-23T16:50:24.513301Z"
    }
   },
   "outputs": [],
   "source": [
    "values = []; indices = []; columns = []\n",
    "\n",
    "for pair in pairs:\n",
    "    indices.append(corrs.index[pair[0]])\n",
    "    columns.append(corrs.columns[pair[1]])\n",
    "    values.append(corrs.values[pair])\n",
    "    \n",
    "subset_df = pd.DataFrame(dict(value=values, var1=indices, var2=columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:50:25.388032Z",
     "start_time": "2019-02-23T16:50:25.352969Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.111172</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.117402</td>\n",
       "      <td>8</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.104640</td>\n",
       "      <td>44</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      value  var1  var2\n",
       "0 -0.111172     3    54\n",
       "1 -0.117402     8    96\n",
       "2 -0.104640    44    45"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:50:35.252569Z",
     "start_time": "2019-02-23T16:50:35.217497Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.11117190190235929"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs.loc[3, 54]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:50:55.128251Z",
     "start_time": "2019-02-23T16:50:55.096675Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.11117190190235929"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs.loc[54, 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:51:03.603309Z",
     "start_time": "2019-02-23T16:51:03.569575Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.11740191658722447"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs.loc[96, 8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:51:12.504120Z",
     "start_time": "2019-02-23T16:51:12.472562Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.10463995106844964"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs.loc[44, 45]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:47:20.066448Z",
     "start_time": "2019-02-23T16:47:20.032246Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.11117190190235929"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs.values[(3, 54)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-23T16:43:58.225875Z",
     "start_time": "2019-02-23T16:43:58.190963Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 54), (8, 96), (44, 45)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[tuple(x) for x in set(map(frozenset, pairs))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

一键复制编辑原始数据按行查看历史

提交于 2019-02-26 21:28 +08:00 . Working

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

https://gitee.com/kaibo_q/Data-Analysis.git

git@gitee.com:kaibo_q/Data-Analysis.git

kaibo_q

Data-Analysis

master

kaibo/Data-Analysis

简介

发行版

贡献者

近期动态

kaibo/Data-Analysis .gitee-modal { width: 500px !important; }

简介

发行版

贡献者

近期动态

搜索帮助

kaibo/Data-Analysis