11-Reading-Writing Data to Different Sources.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11 - Reading/Writing Data to Different Sources - Excel, JSON, SQL, Etc\n",
    "\n",
    "https://youtu.be/N6hyN6BW6ao?si=kFLFwe_sntgN_jim\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "Notes by [Innovinitylabs](https://github.com/innovinitylabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/survey_results_public.csv', index_col='Respondent' )\n",
    "schema_df = pd.read_csv('data/survey_results_schema.csv', index_col='Column')\n",
    "pd.set_option('display.max_columns', 85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.set_option('display.max_rows', 85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>EduOther</th>\n",
       "      <th>OrgSize</th>\n",
       "      <th>DevType</th>\n",
       "      <th>YearsCode</th>\n",
       "      <th>Age1stCode</th>\n",
       "      <th>YearsCodePro</th>\n",
       "      <th>CareerSat</th>\n",
       "      <th>JobSat</th>\n",
       "      <th>MgrIdiot</th>\n",
       "      <th>MgrMoney</th>\n",
       "      <th>MgrWant</th>\n",
       "      <th>JobSeek</th>\n",
       "      <th>LastHireDate</th>\n",
       "      <th>LastInt</th>\n",
       "      <th>FizzBuzz</th>\n",
       "      <th>JobFactors</th>\n",
       "      <th>ResumeUpdate</th>\n",
       "      <th>CurrencySymbol</th>\n",
       "      <th>CurrencyDesc</th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>ConvertedComp</th>\n",
       "      <th>WorkWeekHrs</th>\n",
       "      <th>WorkPlan</th>\n",
       "      <th>WorkChallenge</th>\n",
       "      <th>WorkRemote</th>\n",
       "      <th>WorkLoc</th>\n",
       "      <th>ImpSyn</th>\n",
       "      <th>CodeRev</th>\n",
       "      <th>CodeRevHrs</th>\n",
       "      <th>UnitTests</th>\n",
       "      <th>PurchaseHow</th>\n",
       "      <th>PurchaseWhat</th>\n",
       "      <th>LanguageWorkedWith</th>\n",
       "      <th>LanguageDesireNextYear</th>\n",
       "      <th>DatabaseWorkedWith</th>\n",
       "      <th>DatabaseDesireNextYear</th>\n",
       "      <th>PlatformWorkedWith</th>\n",
       "      <th>PlatformDesireNextYear</th>\n",
       "      <th>WebFrameWorkedWith</th>\n",
       "      <th>WebFrameDesireNextYear</th>\n",
       "      <th>MiscTechWorkedWith</th>\n",
       "      <th>MiscTechDesireNextYear</th>\n",
       "      <th>DevEnviron</th>\n",
       "      <th>OpSys</th>\n",
       "      <th>Containers</th>\n",
       "      <th>BlockchainOrg</th>\n",
       "      <th>BlockchainIs</th>\n",
       "      <th>BetterLife</th>\n",
       "      <th>ITperson</th>\n",
       "      <th>OffOn</th>\n",
       "      <th>SocialMedia</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>SOVisit1st</th>\n",
       "      <th>SOVisitFreq</th>\n",
       "      <th>SOVisitTo</th>\n",
       "      <th>SOFindAnswer</th>\n",
       "      <th>SOTimeSaved</th>\n",
       "      <th>SOHowMuchTime</th>\n",
       "      <th>SOAccount</th>\n",
       "      <th>SOPartFreq</th>\n",
       "      <th>SOJobs</th>\n",
       "      <th>EntTeams</th>\n",
       "      <th>SOComm</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Respondent</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I am a student who is learning to code</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Never</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Not employed, and not looking for work</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>No</td>\n",
       "      <td>Primary/elementary school</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Taught yourself a new language, framework, or ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HTML/CSS;Java;JavaScript;Python</td>\n",
       "      <td>C;C++;C#;Go;HTML/CSS;Java;JavaScript;Python;SQL</td>\n",
       "      <td>SQLite</td>\n",
       "      <td>MySQL</td>\n",
       "      <td>MacOS;Windows</td>\n",
       "      <td>Android;Arduino;Windows</td>\n",
       "      <td>Django;Flask</td>\n",
       "      <td>Flask;jQuery</td>\n",
       "      <td>Node.js</td>\n",
       "      <td>Node.js</td>\n",
       "      <td>IntelliJ;Notepad++;PyCharm</td>\n",
       "      <td>Windows</td>\n",
       "      <td>I do not use containers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Fortunately, someone else has that title</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Twitter</td>\n",
       "      <td>Online</td>\n",
       "      <td>Username</td>\n",
       "      <td>2017</td>\n",
       "      <td>A few times per month or weekly</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>3-5 times per week</td>\n",
       "      <td>Stack Overflow was much faster</td>\n",
       "      <td>31-60 minutes</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No, I didn't know that Stack Overflow had a jo...</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I am a student who is learning to code</td>\n",
       "      <td>No</td>\n",
       "      <td>Less than once per year</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>Bosnia and Herzegovina</td>\n",
       "      <td>Yes, full-time</td>\n",
       "      <td>Secondary school (e.g. American high school, G...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Taken an online course in programming or softw...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Developer, desktop or enterprise applications;...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I am actively looking for a job</td>\n",
       "      <td>I've never had a job</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Financial performance or funding status of the...</td>\n",
       "      <td>Something else changed (education, award, medi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C++;HTML/CSS;Python</td>\n",
       "      <td>C++;HTML/CSS;JavaScript;SQL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MySQL</td>\n",
       "      <td>Windows</td>\n",
       "      <td>Windows</td>\n",
       "      <td>Django</td>\n",
       "      <td>Django</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Atom;PyCharm</td>\n",
       "      <td>Windows</td>\n",
       "      <td>I do not use containers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Useful across many domains and could change ma...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Instagram</td>\n",
       "      <td>Online</td>\n",
       "      <td>Username</td>\n",
       "      <td>2017</td>\n",
       "      <td>Daily or almost daily</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>3-5 times per week</td>\n",
       "      <td>Stack Overflow was much faster</td>\n",
       "      <td>11-30 minutes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A few times per month or weekly</td>\n",
       "      <td>No, I knew that Stack Overflow had a job board...</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        MainBranch Hobbyist  \\\n",
       "Respondent                                                    \n",
       "1           I am a student who is learning to code      Yes   \n",
       "2           I am a student who is learning to code       No   \n",
       "\n",
       "                        OpenSourcer  \\\n",
       "Respondent                            \n",
       "1                             Never   \n",
       "2           Less than once per year   \n",
       "\n",
       "                                                   OpenSource  \\\n",
       "Respondent                                                      \n",
       "1           The quality of OSS and closed source software ...   \n",
       "2           The quality of OSS and closed source software ...   \n",
       "\n",
       "                                        Employment                 Country  \\\n",
       "Respondent                                                                   \n",
       "1           Not employed, and not looking for work          United Kingdom   \n",
       "2               Not employed, but looking for work  Bosnia and Herzegovina   \n",
       "\n",
       "                   Student                                            EdLevel  \\\n",
       "Respondent                                                                      \n",
       "1                       No                          Primary/elementary school   \n",
       "2           Yes, full-time  Secondary school (e.g. American high school, G...   \n",
       "\n",
       "           UndergradMajor                                           EduOther  \\\n",
       "Respondent                                                                     \n",
       "1                     NaN  Taught yourself a new language, framework, or ...   \n",
       "2                     NaN  Taken an online course in programming or softw...   \n",
       "\n",
       "           OrgSize                                            DevType  \\\n",
       "Respondent                                                              \n",
       "1              NaN                                                NaN   \n",
       "2              NaN  Developer, desktop or enterprise applications;...   \n",
       "\n",
       "           YearsCode Age1stCode YearsCodePro CareerSat JobSat MgrIdiot  \\\n",
       "Respondent                                                               \n",
       "1                  4         10          NaN       NaN    NaN      NaN   \n",
       "2                NaN         17          NaN       NaN    NaN      NaN   \n",
       "\n",
       "           MgrMoney MgrWant                          JobSeek  \\\n",
       "Respondent                                                     \n",
       "1               NaN     NaN                              NaN   \n",
       "2               NaN     NaN  I am actively looking for a job   \n",
       "\n",
       "                    LastHireDate LastInt FizzBuzz  \\\n",
       "Respondent                                          \n",
       "1                            NaN     NaN      NaN   \n",
       "2           I've never had a job     NaN      NaN   \n",
       "\n",
       "                                                   JobFactors  \\\n",
       "Respondent                                                      \n",
       "1                                                         NaN   \n",
       "2           Financial performance or funding status of the...   \n",
       "\n",
       "                                                 ResumeUpdate CurrencySymbol  \\\n",
       "Respondent                                                                     \n",
       "1                                                         NaN            NaN   \n",
       "2           Something else changed (education, award, medi...            NaN   \n",
       "\n",
       "           CurrencyDesc  CompTotal CompFreq  ConvertedComp  WorkWeekHrs  \\\n",
       "Respondent                                                                \n",
       "1                   NaN        NaN      NaN            NaN          NaN   \n",
       "2                   NaN        NaN      NaN            NaN          NaN   \n",
       "\n",
       "           WorkPlan WorkChallenge WorkRemote WorkLoc ImpSyn CodeRev  \\\n",
       "Respondent                                                            \n",
       "1               NaN           NaN        NaN     NaN    NaN     NaN   \n",
       "2               NaN           NaN        NaN     NaN    NaN     NaN   \n",
       "\n",
       "            CodeRevHrs UnitTests PurchaseHow PurchaseWhat  \\\n",
       "Respondent                                                  \n",
       "1                  NaN       NaN         NaN          NaN   \n",
       "2                  NaN       NaN         NaN          NaN   \n",
       "\n",
       "                         LanguageWorkedWith  \\\n",
       "Respondent                                    \n",
       "1           HTML/CSS;Java;JavaScript;Python   \n",
       "2                       C++;HTML/CSS;Python   \n",
       "\n",
       "                                     LanguageDesireNextYear  \\\n",
       "Respondent                                                    \n",
       "1           C;C++;C#;Go;HTML/CSS;Java;JavaScript;Python;SQL   \n",
       "2                               C++;HTML/CSS;JavaScript;SQL   \n",
       "\n",
       "           DatabaseWorkedWith DatabaseDesireNextYear PlatformWorkedWith  \\\n",
       "Respondent                                                                \n",
       "1                      SQLite                  MySQL      MacOS;Windows   \n",
       "2                         NaN                  MySQL            Windows   \n",
       "\n",
       "             PlatformDesireNextYear WebFrameWorkedWith WebFrameDesireNextYear  \\\n",
       "Respondent                                                                      \n",
       "1           Android;Arduino;Windows       Django;Flask           Flask;jQuery   \n",
       "2                           Windows             Django                 Django   \n",
       "\n",
       "           MiscTechWorkedWith MiscTechDesireNextYear  \\\n",
       "Respondent                                             \n",
       "1                     Node.js                Node.js   \n",
       "2                         NaN                    NaN   \n",
       "\n",
       "                            DevEnviron    OpSys               Containers  \\\n",
       "Respondent                                                                 \n",
       "1           IntelliJ;Notepad++;PyCharm  Windows  I do not use containers   \n",
       "2                         Atom;PyCharm  Windows  I do not use containers   \n",
       "\n",
       "           BlockchainOrg                                       BlockchainIs  \\\n",
       "Respondent                                                                    \n",
       "1                    NaN                                                NaN   \n",
       "2                    NaN  Useful across many domains and could change ma...   \n",
       "\n",
       "           BetterLife                                  ITperson OffOn  \\\n",
       "Respondent                                                              \n",
       "1                 Yes  Fortunately, someone else has that title   Yes   \n",
       "2                 Yes                                       Yes   Yes   \n",
       "\n",
       "           SocialMedia Extraversion ScreenName SOVisit1st  \\\n",
       "Respondent                                                  \n",
       "1              Twitter       Online   Username       2017   \n",
       "2            Instagram       Online   Username       2017   \n",
       "\n",
       "                                SOVisitFreq  \\\n",
       "Respondent                                    \n",
       "1           A few times per month or weekly   \n",
       "2                     Daily or almost daily   \n",
       "\n",
       "                                                    SOVisitTo  \\\n",
       "Respondent                                                      \n",
       "1           Find answers to specific questions;Learn how t...   \n",
       "2           Find answers to specific questions;Learn how t...   \n",
       "\n",
       "                  SOFindAnswer                     SOTimeSaved  SOHowMuchTime  \\\n",
       "Respondent                                                                      \n",
       "1           3-5 times per week  Stack Overflow was much faster  31-60 minutes   \n",
       "2           3-5 times per week  Stack Overflow was much faster  11-30 minutes   \n",
       "\n",
       "           SOAccount                       SOPartFreq  \\\n",
       "Respondent                                              \n",
       "1                 No                              NaN   \n",
       "2                Yes  A few times per month or weekly   \n",
       "\n",
       "                                                       SOJobs  \\\n",
       "Respondent                                                      \n",
       "1           No, I didn't know that Stack Overflow had a jo...   \n",
       "2           No, I knew that Stack Overflow had a job board...   \n",
       "\n",
       "                                       EntTeams         SOComm  \\\n",
       "Respondent                                                       \n",
       "1           No, and I don't know what those are        Neutral   \n",
       "2           No, and I don't know what those are  Yes, somewhat   \n",
       "\n",
       "                                      WelcomeChange  \\\n",
       "Respondent                                            \n",
       "1           Just as welcome now as I felt last year   \n",
       "2           Just as welcome now as I felt last year   \n",
       "\n",
       "                                                 SONewContent   Age Gender  \\\n",
       "Respondent                                                                   \n",
       "1           Tech articles written by other developers;Indu...  14.0    Man   \n",
       "2           Tech articles written by other developers;Indu...  19.0    Man   \n",
       "\n",
       "           Trans                Sexuality Ethnicity Dependents  \\\n",
       "Respondent                                                       \n",
       "1             No  Straight / Heterosexual       NaN         No   \n",
       "2             No  Straight / Heterosexual       NaN         No   \n",
       "\n",
       "                     SurveyLength                  SurveyEase  \n",
       "Respondent                                                     \n",
       "1           Appropriate in length  Neither easy nor difficult  \n",
       "2           Appropriate in length  Neither easy nor difficult  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CSV & TSV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>to write back to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>EduOther</th>\n",
       "      <th>OrgSize</th>\n",
       "      <th>DevType</th>\n",
       "      <th>YearsCode</th>\n",
       "      <th>Age1stCode</th>\n",
       "      <th>YearsCodePro</th>\n",
       "      <th>CareerSat</th>\n",
       "      <th>JobSat</th>\n",
       "      <th>MgrIdiot</th>\n",
       "      <th>MgrMoney</th>\n",
       "      <th>MgrWant</th>\n",
       "      <th>JobSeek</th>\n",
       "      <th>LastHireDate</th>\n",
       "      <th>LastInt</th>\n",
       "      <th>FizzBuzz</th>\n",
       "      <th>JobFactors</th>\n",
       "      <th>ResumeUpdate</th>\n",
       "      <th>CurrencySymbol</th>\n",
       "      <th>CurrencyDesc</th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>ConvertedComp</th>\n",
       "      <th>WorkWeekHrs</th>\n",
       "      <th>WorkPlan</th>\n",
       "      <th>WorkChallenge</th>\n",
       "      <th>WorkRemote</th>\n",
       "      <th>WorkLoc</th>\n",
       "      <th>ImpSyn</th>\n",
       "      <th>CodeRev</th>\n",
       "      <th>CodeRevHrs</th>\n",
       "      <th>UnitTests</th>\n",
       "      <th>PurchaseHow</th>\n",
       "      <th>PurchaseWhat</th>\n",
       "      <th>LanguageWorkedWith</th>\n",
       "      <th>LanguageDesireNextYear</th>\n",
       "      <th>DatabaseWorkedWith</th>\n",
       "      <th>DatabaseDesireNextYear</th>\n",
       "      <th>PlatformWorkedWith</th>\n",
       "      <th>PlatformDesireNextYear</th>\n",
       "      <th>WebFrameWorkedWith</th>\n",
       "      <th>WebFrameDesireNextYear</th>\n",
       "      <th>MiscTechWorkedWith</th>\n",
       "      <th>MiscTechDesireNextYear</th>\n",
       "      <th>DevEnviron</th>\n",
       "      <th>OpSys</th>\n",
       "      <th>Containers</th>\n",
       "      <th>BlockchainOrg</th>\n",
       "      <th>BlockchainIs</th>\n",
       "      <th>BetterLife</th>\n",
       "      <th>ITperson</th>\n",
       "      <th>OffOn</th>\n",
       "      <th>SocialMedia</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>SOVisit1st</th>\n",
       "      <th>SOVisitFreq</th>\n",
       "      <th>SOVisitTo</th>\n",
       "      <th>SOFindAnswer</th>\n",
       "      <th>SOTimeSaved</th>\n",
       "      <th>SOHowMuchTime</th>\n",
       "      <th>SOAccount</th>\n",
       "      <th>SOPartFreq</th>\n",
       "      <th>SOJobs</th>\n",
       "      <th>EntTeams</th>\n",
       "      <th>SOComm</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Respondent</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>I code primarily as a hobby</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per year</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>India</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>Taught yourself a new language, framework, or ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Developer, back-end;Engineer, site reliability</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...</td>\n",
       "      <td>Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...</td>\n",
       "      <td>Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...</td>\n",
       "      <td>AWS;Docker;Heroku;Linux;MacOS;Slack</td>\n",
       "      <td>Android;Arduino;AWS;Docker;Google Cloud Platfo...</td>\n",
       "      <td>Express;Flask;React.js;Spring</td>\n",
       "      <td>Django;Express;Flask;React.js;Vue.js</td>\n",
       "      <td>Hadoop;Node.js;Pandas</td>\n",
       "      <td>Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...</td>\n",
       "      <td>Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...</td>\n",
       "      <td>Linux-based</td>\n",
       "      <td>Development;Testing;Production;Outside of work...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Useful across many domains and could change ma...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SIGH</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>In real life (in person)</td>\n",
       "      <td>Handle</td>\n",
       "      <td>2012</td>\n",
       "      <td>A few times per week</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>Less than once per week</td>\n",
       "      <td>Stack Overflow was slightly faster</td>\n",
       "      <td>11-30 minutes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per month or monthly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, definitely</td>\n",
       "      <td>A lot more welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Once a month or more often</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>India</td>\n",
       "      <td>No</td>\n",
       "      <td>Master’s degree (MA, MS, M.Eng., MBA, etc.)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10,000 or more employees</td>\n",
       "      <td>Data or business analyst;Data scientist or mac...</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Somewhat confident</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>I’m not actively looking, but I am open to new...</td>\n",
       "      <td>3-4 years ago</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>Languages, frameworks, and other technologies ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INR</td>\n",
       "      <td>Indian rupee</td>\n",
       "      <td>950000.0</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>13293.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>There's no schedule or spec; I work on what se...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A few days each month</td>\n",
       "      <td>Home</td>\n",
       "      <td>Far above average</td>\n",
       "      <td>Yes, because I see value in code review</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Yes, it's part of our process</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C#;Go;JavaScript;Python;R;SQL</td>\n",
       "      <td>C#;Go;JavaScript;Kotlin;Python;R;SQL</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server;MyS...</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server</td>\n",
       "      <td>Linux;Windows</td>\n",
       "      <td>Android;Linux;Raspberry Pi;Windows</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>.NET;Node.js;Pandas;Torch/PyTorch</td>\n",
       "      <td>.NET;Node.js;TensorFlow;Torch/PyTorch</td>\n",
       "      <td>Android Studio;Eclipse;IPython / Jupyter;Notep...</td>\n",
       "      <td>Windows</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Useful for immutable record keeping outside of...</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>Neither</td>\n",
       "      <td>Screen Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Multiple times per day</td>\n",
       "      <td>Find answers to specific questions;Get a sense...</td>\n",
       "      <td>3-5 times per week</td>\n",
       "      <td>They were about the same</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A few times per month or weekly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Somewhat less welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Tech...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Too long</td>\n",
       "      <td>Difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>I am a student who is learning to code</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Never</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>India</td>\n",
       "      <td>Yes, full-time</td>\n",
       "      <td>Secondary school (e.g. American high school, G...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Taken an online course in programming or softw...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Student</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I’m not actively looking, but I am open to new...</td>\n",
       "      <td>I've never had a job</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Industry that I'd be working in;Languages, fra...</td>\n",
       "      <td>Something else changed (education, award, medi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Assembly;Bash/Shell/PowerShell;C;C++;HTML/CSS;...</td>\n",
       "      <td>Assembly;Bash/Shell/PowerShell;C;C++;C#;Go;HTM...</td>\n",
       "      <td>MariaDB;MySQL;Oracle;SQLite</td>\n",
       "      <td>MariaDB;MongoDB;Microsoft SQL Server;MySQL;Ora...</td>\n",
       "      <td>Linux;Windows</td>\n",
       "      <td>Android;Google Cloud Platform;iOS;Linux;MacOS;...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Drupal;jQuer...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.NET;.NET Core;Node.js;TensorFlow;Unity 3D;Unr...</td>\n",
       "      <td>Atom;NetBeans;Notepad++;Sublime Text;Vim</td>\n",
       "      <td>Linux-based</td>\n",
       "      <td>Development</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>What?</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>In real life (in person)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>Daily or almost daily</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>More than 10 times per week</td>\n",
       "      <td>They were about the same</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per month or monthly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, I've heard of them, but I am not part of a...</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Too long</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        MainBranch Hobbyist  \\\n",
       "Respondent                                                    \n",
       "8                      I code primarily as a hobby      Yes   \n",
       "10                  I am a developer by profession      Yes   \n",
       "15          I am a student who is learning to code      Yes   \n",
       "\n",
       "                           OpenSourcer  \\\n",
       "Respondent                               \n",
       "8              Less than once per year   \n",
       "10          Once a month or more often   \n",
       "15                               Never   \n",
       "\n",
       "                                                   OpenSource  \\\n",
       "Respondent                                                      \n",
       "8           OSS is, on average, of HIGHER quality than pro...   \n",
       "10          OSS is, on average, of HIGHER quality than pro...   \n",
       "15          OSS is, on average, of HIGHER quality than pro...   \n",
       "\n",
       "                                    Employment Country         Student  \\\n",
       "Respondent                                                               \n",
       "8           Not employed, but looking for work   India             NaN   \n",
       "10                          Employed full-time   India              No   \n",
       "15          Not employed, but looking for work   India  Yes, full-time   \n",
       "\n",
       "                                                      EdLevel  \\\n",
       "Respondent                                                      \n",
       "8                    Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "10                Master’s degree (MA, MS, M.Eng., MBA, etc.)   \n",
       "15          Secondary school (e.g. American high school, G...   \n",
       "\n",
       "                                               UndergradMajor  \\\n",
       "Respondent                                                      \n",
       "8           Computer science, computer engineering, or sof...   \n",
       "10                                                        NaN   \n",
       "15                                                        NaN   \n",
       "\n",
       "                                                     EduOther  \\\n",
       "Respondent                                                      \n",
       "8           Taught yourself a new language, framework, or ...   \n",
       "10                                                        NaN   \n",
       "15          Taken an online course in programming or softw...   \n",
       "\n",
       "                             OrgSize  \\\n",
       "Respondent                             \n",
       "8                                NaN   \n",
       "10          10,000 or more employees   \n",
       "15                               NaN   \n",
       "\n",
       "                                                      DevType YearsCode  \\\n",
       "Respondent                                                                \n",
       "8              Developer, back-end;Engineer, site reliability         8   \n",
       "10          Data or business analyst;Data scientist or mac...        12   \n",
       "15                                                    Student         3   \n",
       "\n",
       "           Age1stCode YearsCodePro              CareerSat  \\\n",
       "Respondent                                                  \n",
       "8                  16          NaN                    NaN   \n",
       "10                 20           10  Slightly dissatisfied   \n",
       "15                 13          NaN                    NaN   \n",
       "\n",
       "                           JobSat            MgrIdiot MgrMoney MgrWant  \\\n",
       "Respondent                                                               \n",
       "8                             NaN                 NaN      NaN     NaN   \n",
       "10          Slightly dissatisfied  Somewhat confident      Yes     Yes   \n",
       "15                            NaN                 NaN      NaN     NaN   \n",
       "\n",
       "                                                      JobSeek  \\\n",
       "Respondent                                                      \n",
       "8                                                         NaN   \n",
       "10          I’m not actively looking, but I am open to new...   \n",
       "15          I’m not actively looking, but I am open to new...   \n",
       "\n",
       "                    LastHireDate LastInt FizzBuzz  \\\n",
       "Respondent                                          \n",
       "8                            NaN     NaN      NaN   \n",
       "10                 3-4 years ago     NaN       No   \n",
       "15          I've never had a job     NaN      NaN   \n",
       "\n",
       "                                                   JobFactors  \\\n",
       "Respondent                                                      \n",
       "8                                                         NaN   \n",
       "10          Languages, frameworks, and other technologies ...   \n",
       "15          Industry that I'd be working in;Languages, fra...   \n",
       "\n",
       "                                                 ResumeUpdate CurrencySymbol  \\\n",
       "Respondent                                                                     \n",
       "8                                                         NaN            NaN   \n",
       "10                                                        NaN            INR   \n",
       "15          Something else changed (education, award, medi...            NaN   \n",
       "\n",
       "            CurrencyDesc  CompTotal CompFreq  ConvertedComp  WorkWeekHrs  \\\n",
       "Respondent                                                                 \n",
       "8                    NaN        NaN      NaN            NaN          NaN   \n",
       "10          Indian rupee   950000.0   Yearly        13293.0         70.0   \n",
       "15                   NaN        NaN      NaN            NaN          NaN   \n",
       "\n",
       "                                                     WorkPlan WorkChallenge  \\\n",
       "Respondent                                                                    \n",
       "8                                                         NaN           NaN   \n",
       "10          There's no schedule or spec; I work on what se...           NaN   \n",
       "15                                                        NaN           NaN   \n",
       "\n",
       "                       WorkRemote WorkLoc             ImpSyn  \\\n",
       "Respondent                                                     \n",
       "8                             NaN     NaN                NaN   \n",
       "10          A few days each month    Home  Far above average   \n",
       "15                            NaN     NaN                NaN   \n",
       "\n",
       "                                            CodeRev  CodeRevHrs  \\\n",
       "Respondent                                                        \n",
       "8                                               NaN         NaN   \n",
       "10          Yes, because I see value in code review         4.0   \n",
       "15                                              NaN         NaN   \n",
       "\n",
       "                                UnitTests PurchaseHow PurchaseWhat  \\\n",
       "Respondent                                                           \n",
       "8                                     NaN         NaN          NaN   \n",
       "10          Yes, it's part of our process         NaN          NaN   \n",
       "15                                    NaN         NaN          NaN   \n",
       "\n",
       "                                           LanguageWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8           Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...   \n",
       "10                              C#;Go;JavaScript;Python;R;SQL   \n",
       "15          Assembly;Bash/Shell/PowerShell;C;C++;HTML/CSS;...   \n",
       "\n",
       "                                       LanguageDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...   \n",
       "10                       C#;Go;JavaScript;Kotlin;Python;R;SQL   \n",
       "15          Assembly;Bash/Shell/PowerShell;C;C++;C#;Go;HTM...   \n",
       "\n",
       "                                           DatabaseWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8           Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...   \n",
       "10          Elasticsearch;MongoDB;Microsoft SQL Server;MyS...   \n",
       "15                                MariaDB;MySQL;Oracle;SQLite   \n",
       "\n",
       "                                       DatabaseDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...   \n",
       "10                 Elasticsearch;MongoDB;Microsoft SQL Server   \n",
       "15          MariaDB;MongoDB;Microsoft SQL Server;MySQL;Ora...   \n",
       "\n",
       "                             PlatformWorkedWith  \\\n",
       "Respondent                                        \n",
       "8           AWS;Docker;Heroku;Linux;MacOS;Slack   \n",
       "10                                Linux;Windows   \n",
       "15                                Linux;Windows   \n",
       "\n",
       "                                       PlatformDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Android;Arduino;AWS;Docker;Google Cloud Platfo...   \n",
       "10                         Android;Linux;Raspberry Pi;Windows   \n",
       "15          Android;Google Cloud Platform;iOS;Linux;MacOS;...   \n",
       "\n",
       "                                           WebFrameWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8                               Express;Flask;React.js;Spring   \n",
       "10          Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "15                                                        NaN   \n",
       "\n",
       "                                       WebFrameDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8                        Django;Express;Flask;React.js;Vue.js   \n",
       "10          Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "15          Angular/Angular.js;ASP.NET;Django;Drupal;jQuer...   \n",
       "\n",
       "                           MiscTechWorkedWith  \\\n",
       "Respondent                                      \n",
       "8                       Hadoop;Node.js;Pandas   \n",
       "10          .NET;Node.js;Pandas;Torch/PyTorch   \n",
       "15                                        NaN   \n",
       "\n",
       "                                       MiscTechDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...   \n",
       "10                      .NET;Node.js;TensorFlow;Torch/PyTorch   \n",
       "15          .NET;.NET Core;Node.js;TensorFlow;Unity 3D;Unr...   \n",
       "\n",
       "                                                   DevEnviron        OpSys  \\\n",
       "Respondent                                                                   \n",
       "8           Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...  Linux-based   \n",
       "10          Android Studio;Eclipse;IPython / Jupyter;Notep...      Windows   \n",
       "15                   Atom;NetBeans;Notepad++;Sublime Text;Vim  Linux-based   \n",
       "\n",
       "                                                   Containers BlockchainOrg  \\\n",
       "Respondent                                                                    \n",
       "8           Development;Testing;Production;Outside of work...           NaN   \n",
       "10                                                        NaN    Not at all   \n",
       "15                                                Development           NaN   \n",
       "\n",
       "                                                 BlockchainIs BetterLife  \\\n",
       "Respondent                                                                 \n",
       "8           Useful across many domains and could change ma...        Yes   \n",
       "10          Useful for immutable record keeping outside of...         No   \n",
       "15                                                        NaN        Yes   \n",
       "\n",
       "           ITperson  OffOn SocialMedia              Extraversion   ScreenName  \\\n",
       "Respondent                                                                      \n",
       "8              SIGH    Yes     YouTube  In real life (in person)       Handle   \n",
       "10              Yes    Yes     YouTube                   Neither  Screen Name   \n",
       "15              Yes  What?     YouTube  In real life (in person)          NaN   \n",
       "\n",
       "           SOVisit1st             SOVisitFreq  \\\n",
       "Respondent                                      \n",
       "8                2012    A few times per week   \n",
       "10                NaN  Multiple times per day   \n",
       "15               2018   Daily or almost daily   \n",
       "\n",
       "                                                    SOVisitTo  \\\n",
       "Respondent                                                      \n",
       "8           Find answers to specific questions;Learn how t...   \n",
       "10          Find answers to specific questions;Get a sense...   \n",
       "15          Find answers to specific questions;Learn how t...   \n",
       "\n",
       "                           SOFindAnswer                         SOTimeSaved  \\\n",
       "Respondent                                                                    \n",
       "8               Less than once per week  Stack Overflow was slightly faster   \n",
       "10                   3-5 times per week            They were about the same   \n",
       "15          More than 10 times per week            They were about the same   \n",
       "\n",
       "            SOHowMuchTime SOAccount                           SOPartFreq  \\\n",
       "Respondent                                                                 \n",
       "8           11-30 minutes       Yes  Less than once per month or monthly   \n",
       "10                    NaN       Yes      A few times per month or weekly   \n",
       "15                    NaN       Yes  Less than once per month or monthly   \n",
       "\n",
       "           SOJobs                                           EntTeams  \\\n",
       "Respondent                                                             \n",
       "8             Yes                No, and I don't know what those are   \n",
       "10            Yes                No, and I don't know what those are   \n",
       "15            Yes  No, I've heard of them, but I am not part of a...   \n",
       "\n",
       "                     SOComm                             WelcomeChange  \\\n",
       "Respondent                                                              \n",
       "8           Yes, definitely     A lot more welcome now than last year   \n",
       "10            Yes, somewhat  Somewhat less welcome now than last year   \n",
       "15            Yes, somewhat   Just as welcome now as I felt last year   \n",
       "\n",
       "                                                 SONewContent   Age Gender  \\\n",
       "Respondent                                                                   \n",
       "8           Tech articles written by other developers;Indu...  24.0    Man   \n",
       "10          Tech articles written by other developers;Tech...   NaN    NaN   \n",
       "15          Tech articles written by other developers;Indu...  20.0    Man   \n",
       "\n",
       "           Trans                Sexuality Ethnicity Dependents  \\\n",
       "Respondent                                                       \n",
       "8             No  Straight / Heterosexual       NaN        NaN   \n",
       "10           NaN                      NaN       NaN        Yes   \n",
       "15            No                      NaN       NaN        Yes   \n",
       "\n",
       "                     SurveyLength                  SurveyEase  \n",
       "Respondent                                                     \n",
       "8           Appropriate in length  Neither easy nor difficult  \n",
       "10                       Too long                   Difficult  \n",
       "15                       Too long  Neither easy nor difficult  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filt = df['Country'] == 'India'\n",
    "india_df = df[filt]\n",
    "india_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we use `.to_csv()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_csv('data/india_df.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark> Tab delimited file\n",
    "\n",
    "here we need a give an custom separator but use the same `.to_csv()` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_csv('data/india_df.tsv', sep='\\t') # \\t means Tab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>reading a tsv file</mark> we just need to use say the separator while reading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`df = pd.read_csv('data/survey_results_public.csv', index_col='Respondent', sep='\\t' )`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### EXCEL\n",
    "we need the following packages\n",
    "`pip install xlwt openpyxl xlrd`\n",
    "- xlwt - for writing old .xls files\n",
    "- openpyxl - for newer .xlsx\n",
    "- xlrd - for reading Excel files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark> Write Excel files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_excel('data/india_df.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "to read or write to specific sheets we need to include sheet argument"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark> Read excel file</mark>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_excel('data/india_df.xlsx', index_col='Respondent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>EduOther</th>\n",
       "      <th>OrgSize</th>\n",
       "      <th>DevType</th>\n",
       "      <th>YearsCode</th>\n",
       "      <th>Age1stCode</th>\n",
       "      <th>YearsCodePro</th>\n",
       "      <th>CareerSat</th>\n",
       "      <th>JobSat</th>\n",
       "      <th>MgrIdiot</th>\n",
       "      <th>MgrMoney</th>\n",
       "      <th>MgrWant</th>\n",
       "      <th>JobSeek</th>\n",
       "      <th>LastHireDate</th>\n",
       "      <th>LastInt</th>\n",
       "      <th>FizzBuzz</th>\n",
       "      <th>JobFactors</th>\n",
       "      <th>ResumeUpdate</th>\n",
       "      <th>CurrencySymbol</th>\n",
       "      <th>CurrencyDesc</th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>ConvertedComp</th>\n",
       "      <th>WorkWeekHrs</th>\n",
       "      <th>WorkPlan</th>\n",
       "      <th>WorkChallenge</th>\n",
       "      <th>WorkRemote</th>\n",
       "      <th>WorkLoc</th>\n",
       "      <th>ImpSyn</th>\n",
       "      <th>CodeRev</th>\n",
       "      <th>CodeRevHrs</th>\n",
       "      <th>UnitTests</th>\n",
       "      <th>PurchaseHow</th>\n",
       "      <th>PurchaseWhat</th>\n",
       "      <th>LanguageWorkedWith</th>\n",
       "      <th>LanguageDesireNextYear</th>\n",
       "      <th>DatabaseWorkedWith</th>\n",
       "      <th>DatabaseDesireNextYear</th>\n",
       "      <th>PlatformWorkedWith</th>\n",
       "      <th>PlatformDesireNextYear</th>\n",
       "      <th>WebFrameWorkedWith</th>\n",
       "      <th>WebFrameDesireNextYear</th>\n",
       "      <th>MiscTechWorkedWith</th>\n",
       "      <th>MiscTechDesireNextYear</th>\n",
       "      <th>DevEnviron</th>\n",
       "      <th>OpSys</th>\n",
       "      <th>Containers</th>\n",
       "      <th>BlockchainOrg</th>\n",
       "      <th>BlockchainIs</th>\n",
       "      <th>BetterLife</th>\n",
       "      <th>ITperson</th>\n",
       "      <th>OffOn</th>\n",
       "      <th>SocialMedia</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>SOVisit1st</th>\n",
       "      <th>SOVisitFreq</th>\n",
       "      <th>SOVisitTo</th>\n",
       "      <th>SOFindAnswer</th>\n",
       "      <th>SOTimeSaved</th>\n",
       "      <th>SOHowMuchTime</th>\n",
       "      <th>SOAccount</th>\n",
       "      <th>SOPartFreq</th>\n",
       "      <th>SOJobs</th>\n",
       "      <th>EntTeams</th>\n",
       "      <th>SOComm</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Respondent</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>I code primarily as a hobby</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per year</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>India</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>Taught yourself a new language, framework, or ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Developer, back-end;Engineer, site reliability</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...</td>\n",
       "      <td>Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...</td>\n",
       "      <td>Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...</td>\n",
       "      <td>AWS;Docker;Heroku;Linux;MacOS;Slack</td>\n",
       "      <td>Android;Arduino;AWS;Docker;Google Cloud Platfo...</td>\n",
       "      <td>Express;Flask;React.js;Spring</td>\n",
       "      <td>Django;Express;Flask;React.js;Vue.js</td>\n",
       "      <td>Hadoop;Node.js;Pandas</td>\n",
       "      <td>Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...</td>\n",
       "      <td>Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...</td>\n",
       "      <td>Linux-based</td>\n",
       "      <td>Development;Testing;Production;Outside of work...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Useful across many domains and could change ma...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SIGH</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>In real life (in person)</td>\n",
       "      <td>Handle</td>\n",
       "      <td>2012</td>\n",
       "      <td>A few times per week</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>Less than once per week</td>\n",
       "      <td>Stack Overflow was slightly faster</td>\n",
       "      <td>11-30 minutes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per month or monthly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, definitely</td>\n",
       "      <td>A lot more welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Once a month or more often</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>India</td>\n",
       "      <td>No</td>\n",
       "      <td>Master’s degree (MA, MS, M.Eng., MBA, etc.)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10,000 or more employees</td>\n",
       "      <td>Data or business analyst;Data scientist or mac...</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Somewhat confident</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>I’m not actively looking, but I am open to new...</td>\n",
       "      <td>3-4 years ago</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>Languages, frameworks, and other technologies ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INR</td>\n",
       "      <td>Indian rupee</td>\n",
       "      <td>950000.0</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>13293.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>There's no schedule or spec; I work on what se...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A few days each month</td>\n",
       "      <td>Home</td>\n",
       "      <td>Far above average</td>\n",
       "      <td>Yes, because I see value in code review</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Yes, it's part of our process</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C#;Go;JavaScript;Python;R;SQL</td>\n",
       "      <td>C#;Go;JavaScript;Kotlin;Python;R;SQL</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server;MyS...</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server</td>\n",
       "      <td>Linux;Windows</td>\n",
       "      <td>Android;Linux;Raspberry Pi;Windows</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>.NET;Node.js;Pandas;Torch/PyTorch</td>\n",
       "      <td>.NET;Node.js;TensorFlow;Torch/PyTorch</td>\n",
       "      <td>Android Studio;Eclipse;IPython / Jupyter;Notep...</td>\n",
       "      <td>Windows</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Useful for immutable record keeping outside of...</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>Neither</td>\n",
       "      <td>Screen Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Multiple times per day</td>\n",
       "      <td>Find answers to specific questions;Get a sense...</td>\n",
       "      <td>3-5 times per week</td>\n",
       "      <td>They were about the same</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A few times per month or weekly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Somewhat less welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Tech...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Too long</td>\n",
       "      <td>Difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>I am a student who is learning to code</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Never</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>India</td>\n",
       "      <td>Yes, full-time</td>\n",
       "      <td>Secondary school (e.g. American high school, G...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Taken an online course in programming or softw...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Student</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I’m not actively looking, but I am open to new...</td>\n",
       "      <td>I've never had a job</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Industry that I'd be working in;Languages, fra...</td>\n",
       "      <td>Something else changed (education, award, medi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Assembly;Bash/Shell/PowerShell;C;C++;HTML/CSS;...</td>\n",
       "      <td>Assembly;Bash/Shell/PowerShell;C;C++;C#;Go;HTM...</td>\n",
       "      <td>MariaDB;MySQL;Oracle;SQLite</td>\n",
       "      <td>MariaDB;MongoDB;Microsoft SQL Server;MySQL;Ora...</td>\n",
       "      <td>Linux;Windows</td>\n",
       "      <td>Android;Google Cloud Platform;iOS;Linux;MacOS;...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Drupal;jQuer...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>.NET;.NET Core;Node.js;TensorFlow;Unity 3D;Unr...</td>\n",
       "      <td>Atom;NetBeans;Notepad++;Sublime Text;Vim</td>\n",
       "      <td>Linux-based</td>\n",
       "      <td>Development</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>What?</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>In real life (in person)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>Daily or almost daily</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>More than 10 times per week</td>\n",
       "      <td>They were about the same</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per month or monthly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, I've heard of them, but I am not part of a...</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Too long</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        MainBranch Hobbyist  \\\n",
       "Respondent                                                    \n",
       "8                      I code primarily as a hobby      Yes   \n",
       "10                  I am a developer by profession      Yes   \n",
       "15          I am a student who is learning to code      Yes   \n",
       "\n",
       "                           OpenSourcer  \\\n",
       "Respondent                               \n",
       "8              Less than once per year   \n",
       "10          Once a month or more often   \n",
       "15                               Never   \n",
       "\n",
       "                                                   OpenSource  \\\n",
       "Respondent                                                      \n",
       "8           OSS is, on average, of HIGHER quality than pro...   \n",
       "10          OSS is, on average, of HIGHER quality than pro...   \n",
       "15          OSS is, on average, of HIGHER quality than pro...   \n",
       "\n",
       "                                    Employment Country         Student  \\\n",
       "Respondent                                                               \n",
       "8           Not employed, but looking for work   India             NaN   \n",
       "10                          Employed full-time   India              No   \n",
       "15          Not employed, but looking for work   India  Yes, full-time   \n",
       "\n",
       "                                                      EdLevel  \\\n",
       "Respondent                                                      \n",
       "8                    Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "10                Master’s degree (MA, MS, M.Eng., MBA, etc.)   \n",
       "15          Secondary school (e.g. American high school, G...   \n",
       "\n",
       "                                               UndergradMajor  \\\n",
       "Respondent                                                      \n",
       "8           Computer science, computer engineering, or sof...   \n",
       "10                                                        NaN   \n",
       "15                                                        NaN   \n",
       "\n",
       "                                                     EduOther  \\\n",
       "Respondent                                                      \n",
       "8           Taught yourself a new language, framework, or ...   \n",
       "10                                                        NaN   \n",
       "15          Taken an online course in programming or softw...   \n",
       "\n",
       "                             OrgSize  \\\n",
       "Respondent                             \n",
       "8                                NaN   \n",
       "10          10,000 or more employees   \n",
       "15                               NaN   \n",
       "\n",
       "                                                      DevType YearsCode  \\\n",
       "Respondent                                                                \n",
       "8              Developer, back-end;Engineer, site reliability         8   \n",
       "10          Data or business analyst;Data scientist or mac...        12   \n",
       "15                                                    Student         3   \n",
       "\n",
       "           Age1stCode YearsCodePro              CareerSat  \\\n",
       "Respondent                                                  \n",
       "8                  16          NaN                    NaN   \n",
       "10                 20           10  Slightly dissatisfied   \n",
       "15                 13          NaN                    NaN   \n",
       "\n",
       "                           JobSat            MgrIdiot MgrMoney MgrWant  \\\n",
       "Respondent                                                               \n",
       "8                             NaN                 NaN      NaN     NaN   \n",
       "10          Slightly dissatisfied  Somewhat confident      Yes     Yes   \n",
       "15                            NaN                 NaN      NaN     NaN   \n",
       "\n",
       "                                                      JobSeek  \\\n",
       "Respondent                                                      \n",
       "8                                                         NaN   \n",
       "10          I’m not actively looking, but I am open to new...   \n",
       "15          I’m not actively looking, but I am open to new...   \n",
       "\n",
       "                    LastHireDate LastInt FizzBuzz  \\\n",
       "Respondent                                          \n",
       "8                            NaN     NaN      NaN   \n",
       "10                 3-4 years ago     NaN       No   \n",
       "15          I've never had a job     NaN      NaN   \n",
       "\n",
       "                                                   JobFactors  \\\n",
       "Respondent                                                      \n",
       "8                                                         NaN   \n",
       "10          Languages, frameworks, and other technologies ...   \n",
       "15          Industry that I'd be working in;Languages, fra...   \n",
       "\n",
       "                                                 ResumeUpdate CurrencySymbol  \\\n",
       "Respondent                                                                     \n",
       "8                                                         NaN            NaN   \n",
       "10                                                        NaN            INR   \n",
       "15          Something else changed (education, award, medi...            NaN   \n",
       "\n",
       "            CurrencyDesc  CompTotal CompFreq  ConvertedComp  WorkWeekHrs  \\\n",
       "Respondent                                                                 \n",
       "8                    NaN        NaN      NaN            NaN          NaN   \n",
       "10          Indian rupee   950000.0   Yearly        13293.0         70.0   \n",
       "15                   NaN        NaN      NaN            NaN          NaN   \n",
       "\n",
       "                                                     WorkPlan WorkChallenge  \\\n",
       "Respondent                                                                    \n",
       "8                                                         NaN           NaN   \n",
       "10          There's no schedule or spec; I work on what se...           NaN   \n",
       "15                                                        NaN           NaN   \n",
       "\n",
       "                       WorkRemote WorkLoc             ImpSyn  \\\n",
       "Respondent                                                     \n",
       "8                             NaN     NaN                NaN   \n",
       "10          A few days each month    Home  Far above average   \n",
       "15                            NaN     NaN                NaN   \n",
       "\n",
       "                                            CodeRev  CodeRevHrs  \\\n",
       "Respondent                                                        \n",
       "8                                               NaN         NaN   \n",
       "10          Yes, because I see value in code review         4.0   \n",
       "15                                              NaN         NaN   \n",
       "\n",
       "                                UnitTests PurchaseHow PurchaseWhat  \\\n",
       "Respondent                                                           \n",
       "8                                     NaN         NaN          NaN   \n",
       "10          Yes, it's part of our process         NaN          NaN   \n",
       "15                                    NaN         NaN          NaN   \n",
       "\n",
       "                                           LanguageWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8           Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...   \n",
       "10                              C#;Go;JavaScript;Python;R;SQL   \n",
       "15          Assembly;Bash/Shell/PowerShell;C;C++;HTML/CSS;...   \n",
       "\n",
       "                                       LanguageDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...   \n",
       "10                       C#;Go;JavaScript;Kotlin;Python;R;SQL   \n",
       "15          Assembly;Bash/Shell/PowerShell;C;C++;C#;Go;HTM...   \n",
       "\n",
       "                                           DatabaseWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8           Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...   \n",
       "10          Elasticsearch;MongoDB;Microsoft SQL Server;MyS...   \n",
       "15                                MariaDB;MySQL;Oracle;SQLite   \n",
       "\n",
       "                                       DatabaseDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...   \n",
       "10                 Elasticsearch;MongoDB;Microsoft SQL Server   \n",
       "15          MariaDB;MongoDB;Microsoft SQL Server;MySQL;Ora...   \n",
       "\n",
       "                             PlatformWorkedWith  \\\n",
       "Respondent                                        \n",
       "8           AWS;Docker;Heroku;Linux;MacOS;Slack   \n",
       "10                                Linux;Windows   \n",
       "15                                Linux;Windows   \n",
       "\n",
       "                                       PlatformDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Android;Arduino;AWS;Docker;Google Cloud Platfo...   \n",
       "10                         Android;Linux;Raspberry Pi;Windows   \n",
       "15          Android;Google Cloud Platform;iOS;Linux;MacOS;...   \n",
       "\n",
       "                                           WebFrameWorkedWith  \\\n",
       "Respondent                                                      \n",
       "8                               Express;Flask;React.js;Spring   \n",
       "10          Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "15                                                        NaN   \n",
       "\n",
       "                                       WebFrameDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8                        Django;Express;Flask;React.js;Vue.js   \n",
       "10          Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "15          Angular/Angular.js;ASP.NET;Django;Drupal;jQuer...   \n",
       "\n",
       "                           MiscTechWorkedWith  \\\n",
       "Respondent                                      \n",
       "8                       Hadoop;Node.js;Pandas   \n",
       "10          .NET;Node.js;Pandas;Torch/PyTorch   \n",
       "15                                        NaN   \n",
       "\n",
       "                                       MiscTechDesireNextYear  \\\n",
       "Respondent                                                      \n",
       "8           Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...   \n",
       "10                      .NET;Node.js;TensorFlow;Torch/PyTorch   \n",
       "15          .NET;.NET Core;Node.js;TensorFlow;Unity 3D;Unr...   \n",
       "\n",
       "                                                   DevEnviron        OpSys  \\\n",
       "Respondent                                                                   \n",
       "8           Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...  Linux-based   \n",
       "10          Android Studio;Eclipse;IPython / Jupyter;Notep...      Windows   \n",
       "15                   Atom;NetBeans;Notepad++;Sublime Text;Vim  Linux-based   \n",
       "\n",
       "                                                   Containers BlockchainOrg  \\\n",
       "Respondent                                                                    \n",
       "8           Development;Testing;Production;Outside of work...           NaN   \n",
       "10                                                        NaN    Not at all   \n",
       "15                                                Development           NaN   \n",
       "\n",
       "                                                 BlockchainIs BetterLife  \\\n",
       "Respondent                                                                 \n",
       "8           Useful across many domains and could change ma...        Yes   \n",
       "10          Useful for immutable record keeping outside of...         No   \n",
       "15                                                        NaN        Yes   \n",
       "\n",
       "           ITperson  OffOn SocialMedia              Extraversion   ScreenName  \\\n",
       "Respondent                                                                      \n",
       "8              SIGH    Yes     YouTube  In real life (in person)       Handle   \n",
       "10              Yes    Yes     YouTube                   Neither  Screen Name   \n",
       "15              Yes  What?     YouTube  In real life (in person)          NaN   \n",
       "\n",
       "           SOVisit1st             SOVisitFreq  \\\n",
       "Respondent                                      \n",
       "8                2012    A few times per week   \n",
       "10                NaN  Multiple times per day   \n",
       "15               2018   Daily or almost daily   \n",
       "\n",
       "                                                    SOVisitTo  \\\n",
       "Respondent                                                      \n",
       "8           Find answers to specific questions;Learn how t...   \n",
       "10          Find answers to specific questions;Get a sense...   \n",
       "15          Find answers to specific questions;Learn how t...   \n",
       "\n",
       "                           SOFindAnswer                         SOTimeSaved  \\\n",
       "Respondent                                                                    \n",
       "8               Less than once per week  Stack Overflow was slightly faster   \n",
       "10                   3-5 times per week            They were about the same   \n",
       "15          More than 10 times per week            They were about the same   \n",
       "\n",
       "            SOHowMuchTime SOAccount                           SOPartFreq  \\\n",
       "Respondent                                                                 \n",
       "8           11-30 minutes       Yes  Less than once per month or monthly   \n",
       "10                    NaN       Yes      A few times per month or weekly   \n",
       "15                    NaN       Yes  Less than once per month or monthly   \n",
       "\n",
       "           SOJobs                                           EntTeams  \\\n",
       "Respondent                                                             \n",
       "8             Yes                No, and I don't know what those are   \n",
       "10            Yes                No, and I don't know what those are   \n",
       "15            Yes  No, I've heard of them, but I am not part of a...   \n",
       "\n",
       "                     SOComm                             WelcomeChange  \\\n",
       "Respondent                                                              \n",
       "8           Yes, definitely     A lot more welcome now than last year   \n",
       "10            Yes, somewhat  Somewhat less welcome now than last year   \n",
       "15            Yes, somewhat   Just as welcome now as I felt last year   \n",
       "\n",
       "                                                 SONewContent   Age Gender  \\\n",
       "Respondent                                                                   \n",
       "8           Tech articles written by other developers;Indu...  24.0    Man   \n",
       "10          Tech articles written by other developers;Tech...   NaN    NaN   \n",
       "15          Tech articles written by other developers;Indu...  20.0    Man   \n",
       "\n",
       "           Trans                Sexuality Ethnicity Dependents  \\\n",
       "Respondent                                                       \n",
       "8             No  Straight / Heterosexual       NaN        NaN   \n",
       "10           NaN                      NaN       NaN        Yes   \n",
       "15            No                      NaN       NaN        Yes   \n",
       "\n",
       "                     SurveyLength                  SurveyEase  \n",
       "Respondent                                                     \n",
       "8           Appropriate in length  Neither easy nor difficult  \n",
       "10                       Too long                   Difficult  \n",
       "15                       Too long  Neither easy nor difficult  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>write to JSON</mark>\n",
    "\n",
    "we need to use`.to_json()` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_json('data/india_df.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "by default it saves as 'dictionary like' JSON where all the responses are within a 'MainBranch'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "to save it as 'list-like'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_json('data/india_df_list.json', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`lines=True` make each of the records in new line.  \n",
    "also the json is now saved like list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>Read JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we need to use the same arguments (setting) we used to write them when reading them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "json = pd.read_json('data/india_df_list.json', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>EduOther</th>\n",
       "      <th>OrgSize</th>\n",
       "      <th>DevType</th>\n",
       "      <th>YearsCode</th>\n",
       "      <th>Age1stCode</th>\n",
       "      <th>YearsCodePro</th>\n",
       "      <th>CareerSat</th>\n",
       "      <th>JobSat</th>\n",
       "      <th>MgrIdiot</th>\n",
       "      <th>MgrMoney</th>\n",
       "      <th>MgrWant</th>\n",
       "      <th>JobSeek</th>\n",
       "      <th>LastHireDate</th>\n",
       "      <th>LastInt</th>\n",
       "      <th>FizzBuzz</th>\n",
       "      <th>JobFactors</th>\n",
       "      <th>ResumeUpdate</th>\n",
       "      <th>CurrencySymbol</th>\n",
       "      <th>CurrencyDesc</th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>ConvertedComp</th>\n",
       "      <th>WorkWeekHrs</th>\n",
       "      <th>WorkPlan</th>\n",
       "      <th>WorkChallenge</th>\n",
       "      <th>WorkRemote</th>\n",
       "      <th>WorkLoc</th>\n",
       "      <th>ImpSyn</th>\n",
       "      <th>CodeRev</th>\n",
       "      <th>CodeRevHrs</th>\n",
       "      <th>UnitTests</th>\n",
       "      <th>PurchaseHow</th>\n",
       "      <th>PurchaseWhat</th>\n",
       "      <th>LanguageWorkedWith</th>\n",
       "      <th>LanguageDesireNextYear</th>\n",
       "      <th>DatabaseWorkedWith</th>\n",
       "      <th>DatabaseDesireNextYear</th>\n",
       "      <th>PlatformWorkedWith</th>\n",
       "      <th>PlatformDesireNextYear</th>\n",
       "      <th>WebFrameWorkedWith</th>\n",
       "      <th>WebFrameDesireNextYear</th>\n",
       "      <th>MiscTechWorkedWith</th>\n",
       "      <th>MiscTechDesireNextYear</th>\n",
       "      <th>DevEnviron</th>\n",
       "      <th>OpSys</th>\n",
       "      <th>Containers</th>\n",
       "      <th>BlockchainOrg</th>\n",
       "      <th>BlockchainIs</th>\n",
       "      <th>BetterLife</th>\n",
       "      <th>ITperson</th>\n",
       "      <th>OffOn</th>\n",
       "      <th>SocialMedia</th>\n",
       "      <th>Extraversion</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>SOVisit1st</th>\n",
       "      <th>SOVisitFreq</th>\n",
       "      <th>SOVisitTo</th>\n",
       "      <th>SOFindAnswer</th>\n",
       "      <th>SOTimeSaved</th>\n",
       "      <th>SOHowMuchTime</th>\n",
       "      <th>SOAccount</th>\n",
       "      <th>SOPartFreq</th>\n",
       "      <th>SOJobs</th>\n",
       "      <th>EntTeams</th>\n",
       "      <th>SOComm</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I code primarily as a hobby</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per year</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>India</td>\n",
       "      <td>None</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>Taught yourself a new language, framework, or ...</td>\n",
       "      <td>None</td>\n",
       "      <td>Developer, back-end;Engineer, site reliability</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...</td>\n",
       "      <td>Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...</td>\n",
       "      <td>Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...</td>\n",
       "      <td>Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...</td>\n",
       "      <td>AWS;Docker;Heroku;Linux;MacOS;Slack</td>\n",
       "      <td>Android;Arduino;AWS;Docker;Google Cloud Platfo...</td>\n",
       "      <td>Express;Flask;React.js;Spring</td>\n",
       "      <td>Django;Express;Flask;React.js;Vue.js</td>\n",
       "      <td>Hadoop;Node.js;Pandas</td>\n",
       "      <td>Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...</td>\n",
       "      <td>Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...</td>\n",
       "      <td>Linux-based</td>\n",
       "      <td>Development;Testing;Production;Outside of work...</td>\n",
       "      <td>None</td>\n",
       "      <td>Useful across many domains and could change ma...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>SIGH</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>In real life (in person)</td>\n",
       "      <td>Handle</td>\n",
       "      <td>2012</td>\n",
       "      <td>A few times per week</td>\n",
       "      <td>Find answers to specific questions;Learn how t...</td>\n",
       "      <td>Less than once per week</td>\n",
       "      <td>Stack Overflow was slightly faster</td>\n",
       "      <td>11-30 minutes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once per month or monthly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, definitely</td>\n",
       "      <td>A lot more welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Once a month or more often</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>India</td>\n",
       "      <td>No</td>\n",
       "      <td>Master’s degree (MA, MS, M.Eng., MBA, etc.)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10,000 or more employees</td>\n",
       "      <td>Data or business analyst;Data scientist or mac...</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Slightly dissatisfied</td>\n",
       "      <td>Somewhat confident</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>I’m not actively looking, but I am open to new...</td>\n",
       "      <td>3-4 years ago</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "      <td>Languages, frameworks, and other technologies ...</td>\n",
       "      <td>None</td>\n",
       "      <td>INR</td>\n",
       "      <td>Indian rupee</td>\n",
       "      <td>950000.0</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>13293.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>There's no schedule or spec; I work on what se...</td>\n",
       "      <td>None</td>\n",
       "      <td>A few days each month</td>\n",
       "      <td>Home</td>\n",
       "      <td>Far above average</td>\n",
       "      <td>Yes, because I see value in code review</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Yes, it's part of our process</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>C#;Go;JavaScript;Python;R;SQL</td>\n",
       "      <td>C#;Go;JavaScript;Kotlin;Python;R;SQL</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server;MyS...</td>\n",
       "      <td>Elasticsearch;MongoDB;Microsoft SQL Server</td>\n",
       "      <td>Linux;Windows</td>\n",
       "      <td>Android;Linux;Raspberry Pi;Windows</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>Angular/Angular.js;ASP.NET;Django;Express;Flas...</td>\n",
       "      <td>.NET;Node.js;Pandas;Torch/PyTorch</td>\n",
       "      <td>.NET;Node.js;TensorFlow;Torch/PyTorch</td>\n",
       "      <td>Android Studio;Eclipse;IPython / Jupyter;Notep...</td>\n",
       "      <td>Windows</td>\n",
       "      <td>None</td>\n",
       "      <td>Not at all</td>\n",
       "      <td>Useful for immutable record keeping outside of...</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>YouTube</td>\n",
       "      <td>Neither</td>\n",
       "      <td>Screen Name</td>\n",
       "      <td>None</td>\n",
       "      <td>Multiple times per day</td>\n",
       "      <td>Find answers to specific questions;Get a sense...</td>\n",
       "      <td>3-5 times per week</td>\n",
       "      <td>They were about the same</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>A few times per month or weekly</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No, and I don't know what those are</td>\n",
       "      <td>Yes, somewhat</td>\n",
       "      <td>Somewhat less welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Tech...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Too long</td>\n",
       "      <td>Difficult</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       MainBranch Hobbyist                 OpenSourcer  \\\n",
       "0     I code primarily as a hobby      Yes     Less than once per year   \n",
       "1  I am a developer by profession      Yes  Once a month or more often   \n",
       "\n",
       "                                          OpenSource  \\\n",
       "0  OSS is, on average, of HIGHER quality than pro...   \n",
       "1  OSS is, on average, of HIGHER quality than pro...   \n",
       "\n",
       "                           Employment Country Student  \\\n",
       "0  Not employed, but looking for work   India    None   \n",
       "1                  Employed full-time   India      No   \n",
       "\n",
       "                                       EdLevel  \\\n",
       "0     Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "1  Master’s degree (MA, MS, M.Eng., MBA, etc.)   \n",
       "\n",
       "                                      UndergradMajor  \\\n",
       "0  Computer science, computer engineering, or sof...   \n",
       "1                                               None   \n",
       "\n",
       "                                            EduOther  \\\n",
       "0  Taught yourself a new language, framework, or ...   \n",
       "1                                               None   \n",
       "\n",
       "                    OrgSize  \\\n",
       "0                      None   \n",
       "1  10,000 or more employees   \n",
       "\n",
       "                                             DevType YearsCode Age1stCode  \\\n",
       "0     Developer, back-end;Engineer, site reliability         8         16   \n",
       "1  Data or business analyst;Data scientist or mac...        12         20   \n",
       "\n",
       "  YearsCodePro              CareerSat                 JobSat  \\\n",
       "0         None                   None                   None   \n",
       "1           10  Slightly dissatisfied  Slightly dissatisfied   \n",
       "\n",
       "             MgrIdiot MgrMoney MgrWant  \\\n",
       "0                None     None    None   \n",
       "1  Somewhat confident      Yes     Yes   \n",
       "\n",
       "                                             JobSeek   LastHireDate LastInt  \\\n",
       "0                                               None           None    None   \n",
       "1  I’m not actively looking, but I am open to new...  3-4 years ago    None   \n",
       "\n",
       "  FizzBuzz                                         JobFactors ResumeUpdate  \\\n",
       "0     None                                               None         None   \n",
       "1       No  Languages, frameworks, and other technologies ...         None   \n",
       "\n",
       "  CurrencySymbol  CurrencyDesc  CompTotal CompFreq  ConvertedComp  \\\n",
       "0           None          None        NaN     None            NaN   \n",
       "1            INR  Indian rupee   950000.0   Yearly        13293.0   \n",
       "\n",
       "   WorkWeekHrs                                           WorkPlan  \\\n",
       "0          NaN                                               None   \n",
       "1         70.0  There's no schedule or spec; I work on what se...   \n",
       "\n",
       "  WorkChallenge             WorkRemote WorkLoc             ImpSyn  \\\n",
       "0          None                   None    None               None   \n",
       "1          None  A few days each month    Home  Far above average   \n",
       "\n",
       "                                   CodeRev  CodeRevHrs  \\\n",
       "0                                     None         NaN   \n",
       "1  Yes, because I see value in code review         4.0   \n",
       "\n",
       "                       UnitTests PurchaseHow PurchaseWhat  \\\n",
       "0                           None        None         None   \n",
       "1  Yes, it's part of our process        None         None   \n",
       "\n",
       "                                  LanguageWorkedWith  \\\n",
       "0  Bash/Shell/PowerShell;C;C++;HTML/CSS;Java;Java...   \n",
       "1                      C#;Go;JavaScript;Python;R;SQL   \n",
       "\n",
       "                              LanguageDesireNextYear  \\\n",
       "0  Bash/Shell/PowerShell;C;C++;Elixir;Erlang;Go;P...   \n",
       "1               C#;Go;JavaScript;Kotlin;Python;R;SQL   \n",
       "\n",
       "                                  DatabaseWorkedWith  \\\n",
       "0  Cassandra;Elasticsearch;MongoDB;MySQL;Oracle;R...   \n",
       "1  Elasticsearch;MongoDB;Microsoft SQL Server;MyS...   \n",
       "\n",
       "                              DatabaseDesireNextYear  \\\n",
       "0  Cassandra;DynamoDB;Elasticsearch;Firebase;Mong...   \n",
       "1         Elasticsearch;MongoDB;Microsoft SQL Server   \n",
       "\n",
       "                    PlatformWorkedWith  \\\n",
       "0  AWS;Docker;Heroku;Linux;MacOS;Slack   \n",
       "1                        Linux;Windows   \n",
       "\n",
       "                              PlatformDesireNextYear  \\\n",
       "0  Android;Arduino;AWS;Docker;Google Cloud Platfo...   \n",
       "1                 Android;Linux;Raspberry Pi;Windows   \n",
       "\n",
       "                                  WebFrameWorkedWith  \\\n",
       "0                      Express;Flask;React.js;Spring   \n",
       "1  Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "\n",
       "                              WebFrameDesireNextYear  \\\n",
       "0               Django;Express;Flask;React.js;Vue.js   \n",
       "1  Angular/Angular.js;ASP.NET;Django;Express;Flas...   \n",
       "\n",
       "                  MiscTechWorkedWith  \\\n",
       "0              Hadoop;Node.js;Pandas   \n",
       "1  .NET;Node.js;Pandas;Torch/PyTorch   \n",
       "\n",
       "                              MiscTechDesireNextYear  \\\n",
       "0  Ansible;Apache Spark;Chef;Hadoop;Node.js;Panda...   \n",
       "1              .NET;Node.js;TensorFlow;Torch/PyTorch   \n",
       "\n",
       "                                          DevEnviron        OpSys  \\\n",
       "0  Atom;IntelliJ;IPython / Jupyter;PyCharm;Visual...  Linux-based   \n",
       "1  Android Studio;Eclipse;IPython / Jupyter;Notep...      Windows   \n",
       "\n",
       "                                          Containers BlockchainOrg  \\\n",
       "0  Development;Testing;Production;Outside of work...          None   \n",
       "1                                               None    Not at all   \n",
       "\n",
       "                                        BlockchainIs BetterLife ITperson  \\\n",
       "0  Useful across many domains and could change ma...        Yes     SIGH   \n",
       "1  Useful for immutable record keeping outside of...         No      Yes   \n",
       "\n",
       "  OffOn SocialMedia              Extraversion   ScreenName SOVisit1st  \\\n",
       "0   Yes     YouTube  In real life (in person)       Handle       2012   \n",
       "1   Yes     YouTube                   Neither  Screen Name       None   \n",
       "\n",
       "              SOVisitFreq                                          SOVisitTo  \\\n",
       "0    A few times per week  Find answers to specific questions;Learn how t...   \n",
       "1  Multiple times per day  Find answers to specific questions;Get a sense...   \n",
       "\n",
       "              SOFindAnswer                         SOTimeSaved  SOHowMuchTime  \\\n",
       "0  Less than once per week  Stack Overflow was slightly faster  11-30 minutes   \n",
       "1       3-5 times per week            They were about the same           None   \n",
       "\n",
       "  SOAccount                           SOPartFreq SOJobs  \\\n",
       "0       Yes  Less than once per month or monthly    Yes   \n",
       "1       Yes      A few times per month or weekly    Yes   \n",
       "\n",
       "                              EntTeams           SOComm  \\\n",
       "0  No, and I don't know what those are  Yes, definitely   \n",
       "1  No, and I don't know what those are    Yes, somewhat   \n",
       "\n",
       "                              WelcomeChange  \\\n",
       "0     A lot more welcome now than last year   \n",
       "1  Somewhat less welcome now than last year   \n",
       "\n",
       "                                        SONewContent   Age Gender Trans  \\\n",
       "0  Tech articles written by other developers;Indu...  24.0    Man    No   \n",
       "1  Tech articles written by other developers;Tech...   NaN   None  None   \n",
       "\n",
       "                 Sexuality Ethnicity Dependents           SurveyLength  \\\n",
       "0  Straight / Heterosexual      None       None  Appropriate in length   \n",
       "1                     None      None        Yes               Too long   \n",
       "\n",
       "                   SurveyEase  \n",
       "0  Neither easy nor difficult  \n",
       "1                   Difficult  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SQL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-box alert-warning\"> ** NEED AN SQL DB FOR THIS SECTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we need to install sequelalchemy library `sqlalchemy` for this. its popular ORM (object relational mapper) to connect SQL with python \n",
    "\n",
    "`pip install sqlalchemy`\n",
    "\n",
    "for postgres need `pscopg2`\n",
    "\n",
    "`pip install pscopg2-binary `"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sqlalchemy import create_engine\n",
    "import psycopg2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "engine = create_engine('postgresql://dbuser:dbpass@localhost:5432/sample_db')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "database name 'sample_table'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>writing to table in database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "india_df.to_sql('sample_table', engine, if_exists='replace')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark>reading from table in database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql_df = pd.read_sql('sample_table', engine, index_col='Respondent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<mark> to run a specific SQL query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql_df = pd.read_sql_query('SELECT * FROM sample_table', engine, index_col='Respondent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading data from URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts_df = pd.read_json('https://raw.githubusercontent.com/CoreyMSchafer/code_snippets/master/Python/Flask_Blog/snippets/posts.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "---\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}