regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7qf8VLSeYLn-"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.io import arff\n",
    "from io import StringIO\n",
    "import sklearn\n",
    "from sklearn import svm\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.gaussian_process.kernels import RBF, WhiteKernel\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "import warnings\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "warnings.filterwarnings(action='ignore', category=DataConversionWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "class reg4:\n",
    "    def read(self, a):\n",
    "      f = open(a, \"r\")\n",
    "      c = StringIO(f.read())\n",
    "      return np.loadtxt(c, delimiter=',')\n",
    "\n",
    "    def report(self, results, n_top=3):\n",
    "        print(\"\\n\\n\\n\\n\\n\")\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            j = 0\n",
    "            for candidate in candidates:\n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                if j > 1:\n",
    "                    break\n",
    "                j+=1\n",
    "\n",
    "\n",
    "    def randomCV(self, clf, X, y, param_grid, n_iter, cv):\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid,\n",
    "              n_iter = n_iter, cv = cv, iid = False, n_jobs = -1)\n",
    "        random_search.fit(X, y)\n",
    "        self.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "\n",
    "    def Rf(self, x, y):\n",
    "      clf = RandomForestRegressor()\n",
    "      param_grid = {\n",
    "            \"n_estimators\" : np.arange(2,50),\n",
    "            \"max_depth\" : np.arange(1,6),\n",
    "\n",
    "        \"criterion\" : ['mse', 'mae'],\n",
    "        \"min_samples_split\" : np.random.random_sample((100,)),      \n",
    "        \"min_samples_split\" : np.linspace(0.01,1, num = 1000),\n",
    "        \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 100),\n",
    "        \"bootstrap\" : [True, False],\n",
    "        \"warm_start\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 100, 6)  \n",
    "\n",
    "\n",
    "    def Dt(self, x, y):\n",
    "      clf = DecisionTreeRegressor()\n",
    "      param_grid = {\n",
    "          \"max_depth\" : np.arange(1,6),\n",
    "          \"min_samples_split\" : np.linspace(0.01,0.5, num = 1000),\n",
    "          \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 1000),\n",
    "          \"criterion\" : ['mse', 'mae', 'friedman_mse'],\n",
    "          \"splitter\" : ['best', 'random'],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 400, 6)  \n",
    "\n",
    "    def Svr(self, x, y):\n",
    "      clf = svm.SVR()\n",
    "      param_grid = {\n",
    "          \"kernel\" : ['poly', 'rbf', 'linear', 'sigmoid'],\n",
    "          \"gamma\" : ['scale', 'auto'],\n",
    "          \"shrinking\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 15, 6)\n",
    "\n",
    "    def Ada(self, x, y):\n",
    "      clf = AdaBoostRegressor()\n",
    "      param_grid = {\n",
    "          \"n_estimators\" : np.arange(1,100),\n",
    "          \"loss\" : ['linear', 'square', 'exponential'],\n",
    "          # \"learning_rate\" : np.arange(1,)\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 250, 6)\n",
    "\n",
    "    def GP(self, x, y):\n",
    "      clf = GaussianProcessRegressor()\n",
    "      param_grid = {\n",
    "    #       \"kernel\" : ['RBF', 'WhiteKernel'],\n",
    "          \"normalize_y\" : [True, False],\n",
    "          \"copy_X_train\" : [True, False],\n",
    "          \"alpha\" : np.linspace(0, 5, 100),\n",
    "\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 400, 6)\n",
    "\n",
    "    def LR(self, x, y):\n",
    "      clf = LinearRegression()\n",
    "      param_grid = {\n",
    "          \"fit_intercept\" : [True, False],\n",
    "          \"normalize\" : [True, False],\n",
    "          \"copy_X\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 25, 6)\n",
    "\n",
    "    def NN(self, x, y):\n",
    "      clf = MLPRegressor()\n",
    "      param_grid = {\n",
    "          \"hidden_layer_sizes\" : np.arange(1,200),\n",
    "          \"activation\" : ['identity', 'logistic', 'tanh', 'relu'],\n",
    "          \"solver\" : ['lbfgs', 'sgd', 'adam'],\n",
    "          \"learning_rate\" : ['constant', 'invscaling', 'adaptive'],\n",
    "          \"shuffle\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 30, 6)\n",
    "\n",
    "    def start(self):\n",
    "        data = self.read(\"4train.txt\")       # read data\n",
    "        data = data[:,1:]                                 # remove id\n",
    "\n",
    "        # np.random.shuffle(data)                           # shuffle for fairness\n",
    "\n",
    "        y = data[:,-2:-1]                                 # separating prediction var\n",
    "        x = data[:,:data.shape[1] -2]\n",
    "\n",
    "        # temp = data[:,:data.shape[1] -2]\n",
    "\n",
    "        # print(temp.shape)\n",
    "\n",
    "        # x = temp[:,:3].mean(axis = 1)\n",
    "        # # print(x.shape)\n",
    "\n",
    "        # x = np.column_stack((x, temp[:,3:13].mean(axis = 1)))\n",
    "        # x = np.column_stack((x, temp[:,13:17].mean(axis = 1)))\n",
    "        # x = np.column_stack((x, temp[:,17:26].mean(axis = 1)))\n",
    "\n",
    "        # print(x.shape)\n",
    "        x_train, x_test = np.split(x, [940])              # separating test data\n",
    "        y_train, y_test = np.split(y, [940])\n",
    "\n",
    "        sc = SelectKBest(f_regression, k=20).fit(x_train, y_train)\n",
    "        x_train = sc.transform(x_train)\n",
    "        x_test = sc.transform(x_test)\n",
    "\n",
    "\n",
    "        # print(x_test.shape)\n",
    "\n",
    "\n",
    "        scaler = StandardScaler()                         # scaling features\n",
    "        scaler.fit(x_train)\n",
    "        x_train = scaler.transform(x_train)\n",
    "        x_test = scaler.transform(x_test)\n",
    "\n",
    "        # scaler.fit(y_train)\n",
    "        # y_train = scaler.transform(y_train)\n",
    "        # y_test = scaler.transform(y_test)\n",
    "        \n",
    "        # ---------------> \n",
    "        param = self.Dt(x_train, y_train)\n",
    "        reg_tree = DecisionTreeRegressor().set_params(**param)\n",
    "        reg_tree.fit(x_train, y_train)\n",
    "        prediction = reg_tree.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_tree.score(x_test, y_test))\n",
    "        \n",
    "        \n",
    "        # --------------->\n",
    "        param = self.Rf(x_train, y_train)\n",
    "        reg_rf = RandomForestRegressor().set_params(**param)\n",
    "        reg_rf.fit(x_train, y_train)\n",
    "        prediction = reg_rf.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_tree.score(x_test, y_test))\n",
    "        \n",
    "        \n",
    "        # --------------->\n",
    "        param = self.Ada(x_train, y_train)\n",
    "        reg_ada = AdaBoostRegressor().set_params(**param)\n",
    "        reg_ada.fit(x_train, y_train)\n",
    "        prediction = reg_ada.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_ada.score(x_test, y_test))\n",
    "\n",
    "        \n",
    "        # --------------->\n",
    "        param = self.GP(x_train, y_train)\n",
    "        reg_gp = GaussianProcessRegressor().set_params(**param)\n",
    "        reg_gp.fit(x_train, y_train)\n",
    "        prediction = reg_gp.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_gp.score(x_test, y_test))\n",
    "        \n",
    "        \n",
    "        # --------------->\n",
    "        param = self.LR(x_train, y_train)\n",
    "        reg_lr = LinearRegression().set_params(**param)\n",
    "        reg_lr.fit(x_train, y_train)\n",
    "        prediction = reg_lr.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_lr.score(x_test, y_test))\n",
    "\n",
    "        \n",
    "        # # --------------->\n",
    "        param = self.NN(x_train, y_train)\n",
    "        reg_nn = MLPRegressor().set_params(**param)\n",
    "        reg_nn.fit(x_train, y_train)\n",
    "\n",
    "        prediction = reg_nn.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "        print(\"Score with test data\",reg_nn.score(x_test, y_test))\n",
    "        \n",
    "        \n",
    "        # ---------------> Support vector regression\n",
    "        param = self.Svr(x_train, y_train)\n",
    "        reg_nn = svm.SVR().set_params(**param)\n",
    "        reg_nn.fit(x_train, y_train)\n",
    "\n",
    "        prediction = reg_nn.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "        print(\"Score with test data\",reg_nn.score(x_test, y_test))\n",
    "        \n",
    "        \n",
    "class reg5:\n",
    "    def read(self, a):\n",
    "        b = pd.read_csv(a, delimiter = ';')\n",
    "        le = preprocessing.LabelEncoder()\n",
    "        b['Type'] = le.fit_transform(b['Type'])\n",
    "        b = b.astype(np.float64)\n",
    "        return b.to_numpy()\n",
    "\n",
    "    def report(self, results, n_top=3):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            j = 0\n",
    "            for candidate in candidates:\n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                if j > 1:\n",
    "                    break\n",
    "                j+=1\n",
    "\n",
    "\n",
    "\n",
    "    def randomCV(self, clf, X, y, param_grid, n_iter, cv):\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid,\n",
    "              n_iter = n_iter, cv = cv, iid = False)\n",
    "        random_search.fit(X, y)\n",
    "        self.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "\n",
    "    def Rf(self, x, y):\n",
    "      clf = RandomForestRegressor()\n",
    "      param_grid = {\n",
    "            \"n_estimators\" : np.arange(2,50),\n",
    "            \"max_depth\" : np.arange(1,6),\n",
    "\n",
    "        # \"criterion\" : ['mse', 'mae'],\n",
    "        # \"min_samples_split\" : np.random.random_sample((100,)),      \n",
    "        # \"min_samples_split\" : np.linspace(0.01,1, num = 1000),\n",
    "        # \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 100),\n",
    "        # \"bootstrap\" : [True, False],\n",
    "        # \"warm_start\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 50, 6)  \n",
    "\n",
    "\n",
    "    def Dt(self, x, y):\n",
    "      clf = DecisionTreeRegressor()\n",
    "      param_grid = {\n",
    "          \"max_depth\" : np.arange(1,6),\n",
    "          \"min_samples_split\" : np.linspace(0.01,0.5, num = 1000),\n",
    "          \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 1000),\n",
    "\n",
    "          # \"criterion\" : ['mse', 'mae', 'friedman_mse'],\n",
    "          # \"splitter\" : ['best', 'random'],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 400, 6)  \n",
    "\n",
    "    def Svr(self, x, y):\n",
    "      clf = svm.SVR()\n",
    "      param_grid = {\n",
    "          \"kernel\" : ['poly', 'rbf', 'linear', 'sigmoid'],\n",
    "          \"gamma\" : ['scale', 'auto'],\n",
    "          \"shrinking\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def Ada(self, x, y):\n",
    "      clf = AdaBoostRegressor()\n",
    "      param_grid = {\n",
    "          \"n_estimators\" : np.arange(1,100),\n",
    "          \"loss\" : ['linear', 'square', 'exponential'],\n",
    "          # \"learning_rate\" : np.arange(1,)\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 30, 6)\n",
    "\n",
    "    def GP(self, x, y):\n",
    "      clf = GaussianProcessRegressor()\n",
    "      param_grid = {\n",
    "    #       \"kernel\" : [RBF, WhiteKernel],\n",
    "          \"normalize_y\" : [True, False],\n",
    "          \"copy_X_train\" : [True, False],\n",
    "    #       \"alpha\" : np.linspace(0, 3, 100),\n",
    "\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def LR(self, x, y):\n",
    "      clf = LinearRegression()\n",
    "      param_grid = {\n",
    "          \"fit_intercept\" : [True, False],\n",
    "          \"normalize\" : [True, False],\n",
    "          \"copy_X\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 25, 6)\n",
    "\n",
    "    def NN(self, x, y):\n",
    "      clf = MLPRegressor()\n",
    "      param_grid = {\n",
    "          \"hidden_layer_sizes\" : np.arange(1,200),\n",
    "          \"activation\" : ['identity', 'logistic', 'tanh', 'relu'],\n",
    "          \"solver\" : ['lbfgs', 'sgd', 'adam'],\n",
    "          \"learning_rate\" : ['constant', 'invscaling', 'adaptive'],\n",
    "          \"shuffle\" : [True, False],\n",
    "          \"alpha\" : np.random.uniform(0.000001, 1, 1000)\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 30, 6)\n",
    "\n",
    "    def start(self):\n",
    "                # ---------------> Run for data preprocessing\n",
    "\n",
    "        data = self.read(\"5facebook.csv\")\n",
    "\n",
    "        np.random.shuffle(data)\n",
    "\n",
    "        y = data[:,-12:]\n",
    "        x = data[:,:6]\n",
    "\n",
    "        x_test, x_train = np.split(x, [75])\n",
    "        y_test, y_train = np.split(y, [75])\n",
    "\n",
    "        scaler = StandardScaler()                         # scaling features\n",
    "        scaler.fit(x_train)\n",
    "        x_train = scaler.transform(x_train)\n",
    "        x_test = scaler.transform(x_test)\n",
    "\n",
    "        scaler.fit(y_train)\n",
    "        y_train_nn = scaler.transform(y_train)\n",
    "        y_test_nn = scaler.transform(y_test)\n",
    "\n",
    "\n",
    "        # print(y_train.shape, x_train.shape)\n",
    "\n",
    "        # np.where(np.isnan(y_train))\n",
    "\n",
    "        # y_train = np.nan_to_num(y_train)\n",
    "\n",
    "        col_mean = np.nanmean(y_train, axis=0)\n",
    "        inds = np.where(np.isnan(y_train))\n",
    "        y_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_test, axis=0)\n",
    "        inds = np.where(np.isnan(y_test))\n",
    "        y_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(x_train, axis=0)\n",
    "        inds = np.where(np.isnan(x_train))\n",
    "        x_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(x_test, axis=0)\n",
    "        inds = np.where(np.isnan(x_test))\n",
    "        x_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_train_nn, axis=0)\n",
    "        inds = np.where(np.isnan(y_train_nn))\n",
    "        y_train_nn[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_test_nn, axis=0)\n",
    "        inds = np.where(np.isnan(y_test_nn))\n",
    "        y_test_nn[inds] = np.take(col_mean, inds[1])\n",
    "        # ---------------> Run for Decision Tree regresssor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Dt(x_train, i)\n",
    "            reg_tree = DecisionTreeRegressor().set_params(**param)\n",
    "            reg_tree.fit(x_train, i)\n",
    "            prediction = reg_tree.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_tree.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "                    # ---------------> Run for Random Forest regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Rf(x_train, i)\n",
    "            reg_rf = RandomForestRegressor().set_params(**param)\n",
    "            reg_rf.fit(x_train, i)\n",
    "            prediction = reg_rf.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_rf.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "        # ---------------> Run for Support Vector regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Svr(x_train, i)\n",
    "            reg_svr = svm.SVR().set_params(**param)\n",
    "            reg_svr.fit(x_train, i)\n",
    "            prediction = reg_svr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_svr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "                    # ---------------> Run for Adaboost regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Ada(x_train, i)\n",
    "            reg_ada = AdaBoostRegressor().set_params(**param)\n",
    "            reg_ada.fit(x_train, i)\n",
    "            prediction = reg_ada.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_ada.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        # ---------------> Run for Linear regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.LR(x_train, i)\n",
    "            reg_lr = LinearRegression().set_params(**param)\n",
    "            reg_lr.fit(x_train, i)\n",
    "            prediction = reg_lr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_lr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "            # ---------------> Run for Neural Net regressor\n",
    "        j = 0\n",
    "        for i in y_train_nn.T:\n",
    "            param = self.NN(x_train, i)\n",
    "            reg_nn = MLPRegressor().set_params(**param)\n",
    "            reg_nn.fit(x_train, i)\n",
    "            prediction = reg_nn.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test_nn[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_nn.score(x_test, y_test_nn[:,j]))\n",
    "            print('\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        # ---------------> Run for Gaussian Process regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.GP(x_train, i)\n",
    "            reg_gp = GaussianProcessRegressor().set_params(**param)\n",
    "            reg_gp.fit(x_train, i)\n",
    "            prediction = reg_gp.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_gp.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "class reg9:\n",
    "    def read(self, a):\n",
    "        b = pd.read_csv(a, delimiter = ',')\n",
    "        b = b.astype(np.float64)\n",
    "        return b.to_numpy()\n",
    "    def report(self, results, n_top=3):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            j = 0\n",
    "            for candidate in candidates:\n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                if j > 1:\n",
    "                    break\n",
    "                j+=1\n",
    "\n",
    "\n",
    "\n",
    "    def randomCV(self, clf, X, y, param_grid, n_iter, cv):\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid,\n",
    "              n_iter = n_iter, cv = cv, iid = False)\n",
    "        random_search.fit(X, y)\n",
    "        self.report(random_search.cv_results_)\n",
    "        return self.random_search.best_params_\n",
    "\n",
    "    def Rf(self, x, y):\n",
    "      clf = RandomForestRegressor()\n",
    "      param_grid = {\n",
    "            \"n_estimators\" : np.arange(2,50),\n",
    "            \"max_depth\" : np.arange(1,6),\n",
    "\n",
    "        # \"criterion\" : ['mse', 'mae'],\n",
    "        # \"min_samples_split\" : np.random.random_sample((100,)),      \n",
    "        # \"min_samples_split\" : np.linspace(0.01,1, num = 1000),\n",
    "        # \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 100),\n",
    "        # \"bootstrap\" : [True, False],\n",
    "        # \"warm_start\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 50, 6)  \n",
    "\n",
    "\n",
    "    def Dt(self, x, y):\n",
    "      clf = DecisionTreeRegressor()\n",
    "      param_grid = {\n",
    "          \"max_depth\" : np.arange(1,6),\n",
    "    #       \"min_samples_split\" : np.linspace(0.01,0.5, num = 1000),\n",
    "    #       \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 1000),\n",
    "          \"criterion\" : ['mse', 'mae', 'friedman_mse'],\n",
    "          \"splitter\" : ['best', 'random'],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 5, 4)  \n",
    "\n",
    "    def Svr(self, x, y):\n",
    "      clf = svm.SVR()\n",
    "      param_grid = {\n",
    "          \"kernel\" : ['poly', 'rbf', 'linear', 'sigmoid'],\n",
    "          \"gamma\" : ['scale', 'auto'],\n",
    "          \"shrinking\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def Ada(self, x, y):\n",
    "      clf = AdaBoostRegressor()\n",
    "      param_grid = {\n",
    "          \"n_estimators\" : np.arange(1,100),\n",
    "          \"loss\" : ['linear', 'square', 'exponential'],\n",
    "          # \"learning_rate\" : np.arange(1,)\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 30, 6)\n",
    "\n",
    "    def GP(self, x, y):\n",
    "      clf = GaussianProcessRegressor()\n",
    "      param_grid = {\n",
    "    #       \"kernel\" : [RBF, WhiteKernel],\n",
    "          \"normalize_y\" : [True, False],\n",
    "          \"copy_X_train\" : [True, False],\n",
    "          \"alpha\" : np.linspace(0, 3, 100),\n",
    "\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def LR(self, x, y):\n",
    "      clf = LinearRegression()\n",
    "      param_grid = {\n",
    "          \"fit_intercept\" : [True, False],\n",
    "          \"normalize\" : [True, False],\n",
    "          \"copy_X\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 25, 6)\n",
    "\n",
    "    def NN(self, x, y):\n",
    "      clf = MLPRegressor()\n",
    "      param_grid = {\n",
    "          \"hidden_layer_sizes\" : np.arange(1,20),\n",
    "          \"activation\" : ['identity', 'logistic', 'tanh', 'relu'],\n",
    "          \"solver\" : ['lbfgs', 'sgd', 'adam'],\n",
    "          \"learning_rate\" : ['constant', 'invscaling', 'adaptive'],\n",
    "          \"shuffle\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 30, 6)\n",
    "\n",
    "    def start(self):\n",
    "        data = self.read(\"segm.csv\")\n",
    "        np.random.shuffle(data)\n",
    "\n",
    "        y = data[:,-4:]\n",
    "        x = data[:,:13]\n",
    "\n",
    "        x_test, x_train = np.split(x, [36240])\n",
    "        y_test, y_train = np.split(y, [36240])\n",
    "\n",
    "        scaler = StandardScaler()                         # scaling features\n",
    "        scaler.fit(x_train)\n",
    "        x_train = scaler.transform(x_train)\n",
    "        x_test = scaler.transform(x_test)\n",
    "\n",
    "        scaler.fit(y_train)\n",
    "        y_train_nn = scaler.transform(y_train)\n",
    "        y_test_nn = scaler.transform(y_test)\n",
    "\n",
    "\n",
    "        # print(y_train.shape, x_train.shape)\n",
    "\n",
    "        # np.where(np.isnan(y_train))\n",
    "\n",
    "        # y_train = np.nan_to_num(y_train)\n",
    "\n",
    "        col_mean = np.nanmean(y_train, axis=0)\n",
    "        inds = np.where(np.isnan(y_train))\n",
    "        y_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_test, axis=0)\n",
    "        inds = np.where(np.isnan(y_test))\n",
    "        y_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(x_train, axis=0)\n",
    "        inds = np.where(np.isnan(x_train))\n",
    "        x_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(x_test, axis=0)\n",
    "        inds = np.where(np.isnan(x_test))\n",
    "        x_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_train_nn, axis=0)\n",
    "        inds = np.where(np.isnan(y_train_nn))\n",
    "        y_train_nn[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        col_mean = np.nanmean(y_test_nn, axis=0)\n",
    "        inds = np.where(np.isnan(y_test_nn))\n",
    "        y_test_nn[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        x_train = x_train[:20000]\n",
    "        y_train = y_train[:20000]\n",
    "        y_train_nn = y_train_nn[:20000]\n",
    "\n",
    "        x_test = x_test[:3000]\n",
    "        y_test = y_test[:3000]\n",
    "        y_test_nn = y_test_nn[:3000]\n",
    "\n",
    "        # ---------------> Run for Decision Tree regressor\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Dt(x_train, i)\n",
    "            reg_tree = DecisionTreeRegressor().set_params(**param)\n",
    "            reg_tree.fit(x_train, i)\n",
    "            prediction = reg_tree.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_tree.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Rf(x_train, i)\n",
    "            reg_rf = RandomForestRegressor().set_params(**param)\n",
    "            reg_rf.fit(x_train, i)\n",
    "            prediction = reg_rf.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_rf.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Svr(x_train, i)\n",
    "            reg_svr = svm.SVR().set_params(**param)\n",
    "            reg_svr.fit(x_train, i)\n",
    "            prediction = reg_svr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_svr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Ada(x_train, i)\n",
    "            reg_ada = AdaBoostRegressor().set_params(**param)\n",
    "            reg_ada.fit(x_train, i)\n",
    "            prediction = reg_ada.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_ada.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.LR(x_train, i)\n",
    "            reg_lr = LinearRegression().set_params(**param)\n",
    "            reg_lr.fit(x_train, i)\n",
    "            prediction = reg_lr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_lr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "            ########### does not converge ############\n",
    "    #     j = 0\n",
    "    #     for i in y_train.T:\n",
    "    #         param = self.GP(x_train, i)\n",
    "    #         reg_gp = GaussianProcessRegressor().set_params(**param)\n",
    "    #         reg_gp.fit(x_train, i)\n",
    "    #         prediction = reg_gp.predict(x_test)\n",
    "\n",
    "    #         rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "    #         print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "    #         print(\"Score with test data\",reg_gp.score(x_test, y_test[:,j]))\n",
    "    #         print('new data\\n\\n\\n\\n\\n\\n')\n",
    "    #         j+=1\n",
    "\n",
    "        # # --------------->\n",
    "        for i in y_train_nn.T:\n",
    "            print(x_train.shape, i.shape)\n",
    "            param = self.NN(x_train, i)\n",
    "            reg_nn = MLPRegressor().set_params(**param)\n",
    "            reg_nn.fit(x_train, i)\n",
    "\n",
    "            prediction = reg_nn.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "            print(\"Score with test data\",reg_nn.score(x_test, y_test[:,j]))\n",
    "            j+=1        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.io import arff\n",
    "from io import StringIO\n",
    "import sklearn\n",
    "from sklearn import svm\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.gaussian_process.kernels import RBF, WhiteKernel\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "import warnings\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "warnings.filterwarnings(action='ignore', category=DataConversionWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import pandas as pd\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.feature_selection import SelectKBest, f_regression\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.feature_selection import chi2\n",
    "import math\n",
    "\n",
    "class reg10:\n",
    "    def read(self, a):\n",
    "        with open(a) as f:\n",
    "            cols = f.readline().rstrip('\\n').split(',')\n",
    "            X = np.loadtxt(a, delimiter=',', usecols=range(2, len(cols)), skiprows=1, dtype=np.uint8)\n",
    "            y = np.loadtxt(a, delimiter=',', usecols=[1], skiprows=1)\n",
    "        if a == 'ACT2.csv':\n",
    "            np.save('ACT2.npy', np.column_stack((X, y)))\n",
    "        else:\n",
    "            np.save('ACT4.npy', np.column_stack((X, y)))\n",
    "        return np.column_stack((X, y))        \n",
    "    def report(self, results, n_top=3):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            j = 0\n",
    "            for candidate in candidates:\n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                if j > 1:\n",
    "                    break\n",
    "                j+=1\n",
    "\n",
    "\n",
    "\n",
    "    def randomCV(self, clf, X, y, param_grid, n_iter, cv):\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid,\n",
    "              n_iter = n_iter, cv = cv, iid = False, n_jobs = -1)\n",
    "        random_search.fit(X, y)\n",
    "        self.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "\n",
    "    def Rf(self, x, y):\n",
    "      clf = RandomForestRegressor()\n",
    "      param_grid = {\n",
    "            \"n_estimators\" : np.arange(2,50),\n",
    "            \"max_depth\" : np.arange(1,6),\n",
    "\n",
    "        # \"criterion\" : ['mse', 'mae'],\n",
    "        # \"min_samples_split\" : np.random.random_sample((100,)),      \n",
    "        # \"min_samples_split\" : np.linspace(0.01,1, num = 1000),\n",
    "        # \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 100),\n",
    "        # \"bootstrap\" : [True, False],\n",
    "        # \"warm_start\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 5, 6)  \n",
    "\n",
    "\n",
    "    def Dt(self, x, y):\n",
    "      clf = DecisionTreeRegressor()\n",
    "      param_grid = {\n",
    "          \"max_depth\" : np.arange(1,6),\n",
    "    #       \"min_samples_split\" : np.linspace(0.01,0.5, num = 1000),\n",
    "    #       \"min_samples_leaf\" : np.linspace(0.01,0.5, num = 1000),\n",
    "          \"criterion\" : ['mse', 'mae', 'friedman_mse'],\n",
    "          \"splitter\" : ['best', 'random'],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 10, 6)  \n",
    "\n",
    "    def Svr(self, x, y):\n",
    "      clf = svm.SVR()\n",
    "      param_grid = {\n",
    "          \"kernel\" : ['poly', 'rbf', 'linear', 'sigmoid'],\n",
    "          \"gamma\" : ['scale', 'auto'],\n",
    "          \"shrinking\" : [True, False]\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def Ada(self, x, y):\n",
    "      clf = AdaBoostRegressor()\n",
    "      param_grid = {\n",
    "          \"n_estimators\" : np.arange(1,100),\n",
    "          \"loss\" : ['linear', 'square', 'exponential'],\n",
    "          # \"learning_rate\" : np.arange(1,)\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 8, 6)\n",
    "\n",
    "    def GP(self, x, y):\n",
    "      clf = GaussianProcessRegressor()\n",
    "      param_grid = {\n",
    "    #       \"kernel\" : [RBF, WhiteKernel],\n",
    "          \"normalize_y\" : [True, False],\n",
    "          \"copy_X_train\" : [True, False],\n",
    "          \"alpha\" : np.linspace(0, 3, 100),\n",
    "\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 4, 6)\n",
    "\n",
    "    def LR(self, x, y):\n",
    "      clf = LinearRegression()\n",
    "      param_grid = {\n",
    "          \"fit_intercept\" : [True, False],\n",
    "          \"normalize\" : [True, False],\n",
    "          \"copy_X\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 8, 6)\n",
    "\n",
    "    def NN(self, x, y):\n",
    "      clf = MLPRegressor()\n",
    "      param_grid = {\n",
    "          \"hidden_layer_sizes\" : np.arange(1,20),\n",
    "          \"activation\" : ['identity', 'logistic', 'tanh', 'relu'],\n",
    "          \"solver\" : ['lbfgs', 'sgd', 'adam'],\n",
    "          \"learning_rate\" : ['constant', 'invscaling', 'adaptive'],\n",
    "          \"shuffle\" : [True, False],\n",
    "      }\n",
    "      return self.randomCV(clf, x, y, param_grid, 8, 6)\n",
    "\n",
    "    def start(self):\n",
    "        data = self.read(\"ACT2.csv\")\n",
    "        data1 = self.read(\"ACT4.csv\")\n",
    "        \n",
    "#         frames = [data1, data]\n",
    "#         dataC = pd.concat(frames)\n",
    "        \n",
    "#         dataC = dataC.to_numpy()\n",
    "\n",
    "        # le = preprocessing.LabelEncoder()\n",
    "        # j = 0\n",
    "        # for i in dataC.T:\n",
    "        #   if type(i[0]) != np.int or np.float64 or np.int32:\n",
    "        #     i = le.fit_transform(i)\n",
    "\n",
    "        dataC = dataC.astype(np.float64)\n",
    "        col_mean = np.nanmean(dataC, axis=0)\n",
    "        inds = np.where(np.isnan(dataC))\n",
    "        dataC[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        np.random.shuffle(dataC)\n",
    "\n",
    "        y = dataC[:,:1]\n",
    "        x = dataC[:,1:]\n",
    "\n",
    "        p = pd.DataFrame(x)\n",
    "\n",
    "        selector = VarianceThreshold(p.var(axis = 1).mean() * 0.5)\n",
    "\n",
    "        p = selector.fit_transform(p)\n",
    "\n",
    "        x = p\n",
    "        scaler = StandardScaler()                         # scaling features\n",
    "        scaler.fit(x)\n",
    "        x = scaler.transform(x)\n",
    "\n",
    "        pca = PCA()\n",
    "        pca.fit_transform(x_train)\n",
    "        print(len(pca.explained_variance_ratio_[pca.explained_variance_ratio_ > 0.001]))\n",
    "\n",
    "\n",
    "        if x.shape[1] > int(x.shape[1] * 0.5):\n",
    "          x = SelectKBest(k=int(x.shape[1] * 0.5), score_func = f_regression).fit_transform(x, y)\n",
    "\n",
    "        print(x.shape)\n",
    "\n",
    "        pca = PCA(n_components= 52)\n",
    "        x = pca.fit_transform(x)\n",
    "        # x_train = pca.transform(x_train)\n",
    "        # x_test = pca.transform(x_test)\n",
    "\n",
    "\n",
    "        x_test, x_train = np.split(x, [1843])\n",
    "        y_test, y_train = np.split(y, [1843])\n",
    "\n",
    "\n",
    "        # scaler.fit(y_train)\n",
    "        # y_train_nn = scaler.transform(y_train)\n",
    "        # y_test_nn = scaler.transform(y_test)\n",
    "\n",
    "\n",
    "        # print(y_train.shape, x_train.shape)\n",
    "\n",
    "        # np.where(np.isnan(y_train))\n",
    "\n",
    "        # y_train = np.nan_to_num(y_train)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        # col_mean = np.nanmean(y_train, axis=0)\n",
    "        # inds = np.where(np.isnan(y_train))\n",
    "        # y_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # col_mean = np.nanmean(y_test, axis=0)\n",
    "        # inds = np.where(np.isnan(y_test))\n",
    "        # y_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # col_mean = np.nanmean(x_train, axis=0)\n",
    "        # inds = np.where(np.isnan(x_train))\n",
    "        # x_train[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # col_mean = np.nanmean(x_test, axis=0)\n",
    "        # inds = np.where(np.isnan(x_test))\n",
    "        # x_test[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # col_mean = np.nanmean(y_train_nn, axis=0)\n",
    "        # inds = np.where(np.isnan(y_train_nn))\n",
    "        # y_train_nn[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # col_mean = np.nanmean(y_test_nn, axis=0)\n",
    "        # inds = np.where(np.isnan(y_test_nn))\n",
    "        # y_test_nn[inds] = np.take(col_mean, inds[1])\n",
    "\n",
    "        # ---------------> Run for Decision Tree regressor\n",
    "        j = 0\n",
    "        # for i in y_train.T:\n",
    "        param = self.Dt(x_train, y_train)\n",
    "        reg_tree = DecisionTreeRegressor().set_params(**param)\n",
    "        reg_tree.fit(x_train, y_train)\n",
    "        prediction = reg_tree.predict(x_test)\n",
    "\n",
    "        rmse = mean_squared_error(y_test, prediction)\n",
    "        print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "        print(\"Score with test data\",reg_tree.score(x_test, y_test))\n",
    "        print('new data\\n\\n\\n\\n\\n\\n')\n",
    "        j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Rf(x_train, i)\n",
    "            reg_rf = RandomForestRegressor().set_params(**param)\n",
    "            reg_rf.fit(x_train, i)\n",
    "            prediction = reg_rf.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_rf.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Svr(x_train, i)\n",
    "            reg_svr = svm.SVR().set_params(**param)\n",
    "            reg_svr.fit(x_train, i)\n",
    "            prediction = reg_svr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_svr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.Ada(x_train, i)\n",
    "            reg_ada = AdaBoostRegressor().set_params(**param)\n",
    "            reg_ada.fit(x_train, i)\n",
    "            prediction = reg_ada.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_ada.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        for i in y_train.T:\n",
    "            param = self.LR(x_train, i)\n",
    "            reg_lr = LinearRegression().set_params(**param)\n",
    "            reg_lr.fit(x_train, i)\n",
    "            prediction = reg_lr.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_lr.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n",
    "        j = 0\n",
    "        ########## may not converge #############\n",
    "        for i in y_train.T:\n",
    "            param = self.GP(x_train, i)\n",
    "            reg_gp = GaussianProcessRegressor().set_params(**param)\n",
    "            reg_gp.fit(x_train, i)\n",
    "            prediction = reg_gp.predict(x_test)\n",
    "\n",
    "            rmse = mean_squared_error(y_test[:,j], prediction)\n",
    "            print(\"RMSE on test data : \", rmse)\n",
    "\n",
    "            print(\"Score with test data\",reg_gp.score(x_test, y_test[:,j]))\n",
    "            print('new data\\n\\n\\n\\n\\n\\n')\n",
    "            j+=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'ACT2.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-4cc29e66591a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreg10\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-13-13612a321b1d>\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ACT2.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m         \u001b[0mdata1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ACT4.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-13612a321b1d>\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, a)\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mreg10\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m             \u001b[0mcols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadtxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelimiter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0musecols\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'ACT2.csv'"
     ]
    }
   ],
   "source": [
    "c = reg10()\n",
    "c.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: -0.073 (std: 0.207)\n",
      "Parameters: {'min_samples_split': 0.44947947947947947, 'min_samples_leaf': 0.014414414414414415, 'max_depth': 5}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.080 (std: 0.220)\n",
      "Parameters: {'min_samples_split': 0.49656656656656656, 'min_samples_leaf': 0.014904904904904905, 'max_depth': 5}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.083 (std: 0.170)\n",
      "Parameters: {'min_samples_split': 0.2621121121121121, 'min_samples_leaf': 0.499019019019019, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.083 (std: 0.170)\n",
      "Parameters: {'min_samples_split': 0.13997997997998, 'min_samples_leaf': 0.5, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.083 (std: 0.170)\n",
      "Parameters: {'min_samples_split': 0.12575575575575576, 'min_samples_leaf': 0.499019019019019, 'max_depth': 4}\n",
      "\n",
      "RMSE on test data :  382754582.20620346\n",
      "Score with test data 0.03560677049618999\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.129 (std: 0.258)\n",
      "Parameters: {'min_samples_split': 0.01, 'min_samples_leaf': 0.4269169169169169, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.135 (std: 0.256)\n",
      "Parameters: {'min_samples_split': 0.2101201201201201, 'min_samples_leaf': 0.42397397397397396, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.135 (std: 0.256)\n",
      "Parameters: {'min_samples_split': 0.27045045045045046, 'min_samples_leaf': 0.422992992992993, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.135 (std: 0.256)\n",
      "Parameters: {'min_samples_split': 0.3082182182182182, 'min_samples_leaf': 0.4166166166166166, 'max_depth': 5}\n",
      "\n",
      "RMSE on test data :  1433606185.276025\n",
      "Score with test data 0.003917706202904769\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.231 (std: 0.147)\n",
      "Parameters: {'min_samples_split': 0.2415115115115115, 'min_samples_leaf': 0.09338338338338337, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.226 (std: 0.140)\n",
      "Parameters: {'min_samples_split': 0.11006006006006004, 'min_samples_leaf': 0.05463463463463463, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.225 (std: 0.139)\n",
      "Parameters: {'min_samples_split': 0.3038038038038038, 'min_samples_leaf': 0.017357357357357356, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.225 (std: 0.139)\n",
      "Parameters: {'min_samples_split': 0.16205205205205206, 'min_samples_leaf': 0.0919119119119119, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.225 (std: 0.139)\n",
      "Parameters: {'min_samples_split': 0.13262262262262262, 'min_samples_leaf': 0.06395395395395395, 'max_depth': 3}\n",
      "\n",
      "RMSE on test data :  1715798.9303213782\n",
      "Score with test data 0.0833724435860862\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.311 (std: 0.168)\n",
      "Parameters: {'min_samples_split': 0.11447447447447447, 'min_samples_leaf': 0.017357357357357356, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.310 (std: 0.168)\n",
      "Parameters: {'min_samples_split': 0.21208208208208207, 'min_samples_leaf': 0.05267267267267267, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.310 (std: 0.168)\n",
      "Parameters: {'min_samples_split': 0.17088088088088088, 'min_samples_leaf': 0.046296296296296294, 'max_depth': 4}\n",
      "\n",
      "RMSE on test data :  1622392.6058920086\n",
      "Score with test data 0.10647406279901506\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.121 (std: 0.088)\n",
      "Parameters: {'min_samples_split': 0.24788788788788788, 'min_samples_leaf': 0.05071071071071071, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.121 (std: 0.088)\n",
      "Parameters: {'min_samples_split': 0.03011011011011011, 'min_samples_leaf': 0.09436436436436435, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.121 (std: 0.088)\n",
      "Parameters: {'min_samples_split': 0.37149149149149147, 'min_samples_leaf': 0.09534534534534533, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  7552983.01254456\n",
      "Score with test data -0.0035428573062634694\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.088 (std: 0.124)\n",
      "Parameters: {'min_samples_split': 0.07180180180180179, 'min_samples_leaf': 0.499019019019019, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.088 (std: 0.124)\n",
      "Parameters: {'min_samples_split': 0.267017017017017, 'min_samples_leaf': 0.499019019019019, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.088 (std: 0.124)\n",
      "Parameters: {'min_samples_split': 0.31900900900900897, 'min_samples_leaf': 0.4985285285285285, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  532945815.7907896\n",
      "Score with test data -0.025672612161478536\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.054 (std: 0.081)\n",
      "Parameters: {'min_samples_split': 0.36756756756756753, 'min_samples_leaf': 0.08014014014014013, 'max_depth': 5}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.051 (std: 0.086)\n",
      "Parameters: {'min_samples_split': 0.4622322322322322, 'min_samples_leaf': 0.037467467467467465, 'max_depth': 5}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.051 (std: 0.086)\n",
      "Parameters: {'min_samples_split': 0.44555555555555554, 'min_samples_leaf': 0.053653653653653655, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.051 (std: 0.086)\n",
      "Parameters: {'min_samples_split': 0.3881681681681681, 'min_samples_leaf': 0.04923923923923924, 'max_depth': 4}\n",
      "\n",
      "RMSE on test data :  43335283.33964098\n",
      "Score with test data -0.025102856112214633\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.283 (std: 0.173)\n",
      "Parameters: {'min_samples_split': 0.294974974974975, 'min_samples_leaf': 0.010980980980980982, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.283 (std: 0.173)\n",
      "Parameters: {'min_samples_split': 0.32685685685685684, 'min_samples_leaf': 0.013433433433433433, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.275 (std: 0.174)\n",
      "Parameters: {'min_samples_split': 0.49019019019019017, 'min_samples_leaf': 0.03158158158158158, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.275 (std: 0.174)\n",
      "Parameters: {'min_samples_split': 0.3356856856856857, 'min_samples_leaf': 0.08210210210210209, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.275 (std: 0.174)\n",
      "Parameters: {'min_samples_split': 0.21943943943943944, 'min_samples_leaf': 0.021771771771771774, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  165248.54124316547\n",
      "Score with test data 0.35789641962538643\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.029 (std: 0.027)\n",
      "Parameters: {'min_samples_split': 0.45144144144144144, 'min_samples_leaf': 0.4985285285285285, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.031 (std: 0.044)\n",
      "Parameters: {'min_samples_split': 0.27437437437437434, 'min_samples_leaf': 0.4612512512512512, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.031 (std: 0.044)\n",
      "Parameters: {'min_samples_split': 0.22238238238238237, 'min_samples_leaf': 0.4617417417417417, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  95.04646828143024\n",
      "Score with test data -0.02776345763896715\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.003 (std: 0.037)\n",
      "Parameters: {'min_samples_split': 0.13850850850850852, 'min_samples_leaf': 0.2052152152152152, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.003 (std: 0.038)\n",
      "Parameters: {'min_samples_split': 0.3763963963963964, 'min_samples_leaf': 0.20815815815815816, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.006 (std: 0.034)\n",
      "Parameters: {'min_samples_split': 0.4931331331331331, 'min_samples_leaf': 0.21698698698698698, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  42909.65096197127\n",
      "Score with test data 0.04192548659918094\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.037 (std: 0.055)\n",
      "Parameters: {'min_samples_split': 0.4985285285285285, 'min_samples_leaf': 0.4028828828828829, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.037 (std: 0.055)\n",
      "Parameters: {'min_samples_split': 0.3822822822822823, 'min_samples_leaf': 0.3494194194194194, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.037 (std: 0.055)\n",
      "Parameters: {'min_samples_split': 0.3597197197197197, 'min_samples_leaf': 0.4009209209209209, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  673.1069898633438\n",
      "Score with test data 0.13818472926115288\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.001 (std: 0.039)\n",
      "Parameters: {'min_samples_split': 0.12967967967967967, 'min_samples_leaf': 0.20864864864864865, 'max_depth': 4}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.000 (std: 0.036)\n",
      "Parameters: {'min_samples_split': 0.2066866866866867, 'min_samples_leaf': 0.2061961961961962, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.004 (std: 0.035)\n",
      "Parameters: {'min_samples_split': 0.2959559559559559, 'min_samples_leaf': 0.22091091091091092, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.004 (std: 0.035)\n",
      "Parameters: {'min_samples_split': 0.03011011011011011, 'min_samples_leaf': 0.22140140140140138, 'max_depth': 3}\n",
      "\n",
      "RMSE on test data :  56328.3256074301\n",
      "Score with test data 0.04533870991596933\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: -0.084 (std: 0.244)\n",
      "Parameters: {'n_estimators': 43, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.091 (std: 0.279)\n",
      "Parameters: {'n_estimators': 47, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.092 (std: 0.249)\n",
      "Parameters: {'n_estimators': 32, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  403147049.39608264\n",
      "Score with test data -0.01577434472767192\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.205 (std: 0.396)\n",
      "Parameters: {'n_estimators': 27, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.205 (std: 0.466)\n",
      "Parameters: {'n_estimators': 15, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.207 (std: 0.408)\n",
      "Parameters: {'n_estimators': 44, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  1578906638.3940132\n",
      "Score with test data -0.09703833745678159\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.199 (std: 0.111)\n",
      "Parameters: {'n_estimators': 27, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.194 (std: 0.134)\n",
      "Parameters: {'n_estimators': 12, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.186 (std: 0.121)\n",
      "Parameters: {'n_estimators': 20, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  1794606.076643126\n",
      "Score with test data 0.04127147203032977\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.272 (std: 0.158)\n",
      "Parameters: {'n_estimators': 21, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.261 (std: 0.137)\n",
      "Parameters: {'n_estimators': 29, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.259 (std: 0.127)\n",
      "Parameters: {'n_estimators': 35, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  1680408.423813619\n",
      "Score with test data 0.07452209390281361\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.103 (std: 0.098)\n",
      "Parameters: {'n_estimators': 41, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.091 (std: 0.097)\n",
      "Parameters: {'n_estimators': 45, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.090 (std: 0.097)\n",
      "Parameters: {'n_estimators': 28, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  7326392.419618503\n",
      "Score with test data 0.02656357490549488\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.152 (std: 0.232)\n",
      "Parameters: {'n_estimators': 10, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.214 (std: 0.276)\n",
      "Parameters: {'n_estimators': 16, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.220 (std: 0.385)\n",
      "Parameters: {'n_estimators': 26, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  598918309.7350894\n",
      "Score with test data -0.1526389531848218\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.047 (std: 0.068)\n",
      "Parameters: {'n_estimators': 18, 'max_depth': 3}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.044 (std: 0.060)\n",
      "Parameters: {'n_estimators': 47, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.043 (std: 0.068)\n",
      "Parameters: {'n_estimators': 28, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  45910503.785167366\n",
      "Score with test data -0.08602009560820845\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.251 (std: 0.180)\n",
      "Parameters: {'n_estimators': 36, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.251 (std: 0.167)\n",
      "Parameters: {'n_estimators': 2, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.250 (std: 0.176)\n",
      "Parameters: {'n_estimators': 17, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  152953.94584113598\n",
      "Score with test data 0.4056693298581229\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.049 (std: 0.122)\n",
      "Parameters: {'n_estimators': 44, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.050 (std: 0.109)\n",
      "Parameters: {'n_estimators': 34, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.055 (std: 0.115)\n",
      "Parameters: {'n_estimators': 38, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  102.5888606486925\n",
      "Score with test data -0.10932140922212708\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.006 (std: 0.027)\n",
      "Parameters: {'n_estimators': 9, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.011 (std: 0.026)\n",
      "Parameters: {'n_estimators': 39, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.011 (std: 0.027)\n",
      "Parameters: {'n_estimators': 8, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  43180.23477005134\n",
      "Score with test data 0.03588396809579786\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.038 (std: 0.080)\n",
      "Parameters: {'n_estimators': 42, 'max_depth': 2}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.037 (std: 0.055)\n",
      "Parameters: {'n_estimators': 18, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.034 (std: 0.072)\n",
      "Parameters: {'n_estimators': 47, 'max_depth': 2}\n",
      "\n",
      "RMSE on test data :  687.9766359416936\n",
      "Score with test data 0.11914631745769544\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.006 (std: 0.025)\n",
      "Parameters: {'n_estimators': 28, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.007 (std: 0.027)\n",
      "Parameters: {'n_estimators': 43, 'max_depth': 1}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.008 (std: 0.031)\n",
      "Parameters: {'n_estimators': 17, 'max_depth': 1}\n",
      "\n",
      "RMSE on test data :  56905.64294305094\n",
      "Score with test data 0.03555424523557482\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.156 (std: 0.036)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.157 (std: 0.036)\n",
      "Parameters: {'shrinking': True, 'kernel': 'rbf', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.158 (std: 0.036)\n",
      "Parameters: {'shrinking': False, 'kernel': 'sigmoid', 'gamma': 'scale'}\n",
      "\n",
      "RMSE on test data :  464013087.22999084\n",
      "Score with test data -0.16913317443887488\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.112 (std: 0.041)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.112 (std: 0.041)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.112 (std: 0.041)\n",
      "Parameters: {'shrinking': True, 'kernel': 'rbf', 'gamma': 'scale'}\n",
      "\n",
      "RMSE on test data :  1689443451.3988008\n",
      "Score with test data -0.173840295607953\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.080 (std: 0.056)\n",
      "Parameters: {'shrinking': True, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.080 (std: 0.056)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.080 (std: 0.056)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  2005004.5042760947\n",
      "Score with test data -0.0711292255026934\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.007 (std: 0.058)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.071 (std: 0.054)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.071 (std: 0.055)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  1856045.3875325469\n",
      "Score with test data -0.02220922874014497\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.049 (std: 0.051)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.049 (std: 0.051)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.082 (std: 0.047)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  8158319.443409198\n",
      "Score with test data -0.08397214603265479\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.116 (std: 0.077)\n",
      "Parameters: {'shrinking': True, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.117 (std: 0.077)\n",
      "Parameters: {'shrinking': True, 'kernel': 'sigmoid', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.117 (std: 0.077)\n",
      "Parameters: {'shrinking': False, 'kernel': 'sigmoid', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  578018230.8331783\n",
      "Score with test data -0.11241603016609746\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.174 (std: 0.067)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.180 (std: 0.067)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.182 (std: 0.069)\n",
      "Parameters: {'shrinking': False, 'kernel': 'sigmoid', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  51890525.211344965\n",
      "Score with test data -0.22747843096837794\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: -0.033 (std: 0.049)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.033 (std: 0.049)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.092 (std: 0.049)\n",
      "Parameters: {'shrinking': True, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "RMSE on test data :  264906.30987689673\n",
      "Score with test data -0.029342157916430796\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.083 (std: 0.035)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.083 (std: 0.035)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.083 (std: 0.035)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  104.1488601489832\n",
      "Score with test data -0.1261901104934513\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.056 (std: 0.039)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.056 (std: 0.039)\n",
      "Parameters: {'shrinking': True, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.067 (std: 0.041)\n",
      "Parameters: {'shrinking': False, 'kernel': 'sigmoid', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  47853.00478197694\n",
      "Score with test data -0.06844831508630445\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.009 (std: 0.075)\n",
      "Parameters: {'shrinking': False, 'kernel': 'linear', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.006 (std: 0.065)\n",
      "Parameters: {'shrinking': True, 'kernel': 'rbf', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.022 (std: 0.066)\n",
      "Parameters: {'shrinking': True, 'kernel': 'sigmoid', 'gamma': 'auto'}\n",
      "\n",
      "RMSE on test data :  763.7265964811988\n",
      "Score with test data 0.022159547547514372\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.071 (std: 0.045)\n",
      "Parameters: {'shrinking': True, 'kernel': 'rbf', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.071 (std: 0.045)\n",
      "Parameters: {'shrinking': False, 'kernel': 'rbf', 'gamma': 'auto'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.077 (std: 0.041)\n",
      "Parameters: {'shrinking': False, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.077 (std: 0.041)\n",
      "Parameters: {'shrinking': True, 'kernel': 'poly', 'gamma': 'scale'}\n",
      "\n",
      "RMSE on test data :  64158.84049028569\n",
      "Score with test data -0.08737408350502052\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.319 (std: 0.542)\n",
      "Parameters: {'n_estimators': 2, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.382 (std: 0.641)\n",
      "Parameters: {'n_estimators': 4, 'loss': 'linear'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.758 (std: 1.215)\n",
      "Parameters: {'n_estimators': 1, 'loss': 'linear'}\n",
      "\n",
      "RMSE on test data :  762130743.7631975\n",
      "Score with test data -0.9202741481118666\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.700 (std: 0.628)\n",
      "Parameters: {'n_estimators': 14, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.727 (std: 0.764)\n",
      "Parameters: {'n_estimators': 36, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.937 (std: 0.927)\n",
      "Parameters: {'n_estimators': 84, 'loss': 'square'}\n",
      "\n",
      "RMSE on test data :  2615762428.4399757\n",
      "Score with test data -0.8174549374220843\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.066 (std: 0.091)\n",
      "Parameters: {'n_estimators': 19, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.048 (std: 0.097)\n",
      "Parameters: {'n_estimators': 23, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.014 (std: 0.185)\n",
      "Parameters: {'n_estimators': 5, 'loss': 'square'}\n",
      "\n",
      "RMSE on test data :  1954208.9264867047\n",
      "Score with test data -0.04399281369889274\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.215 (std: 0.136)\n",
      "Parameters: {'n_estimators': 1, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.176 (std: 0.105)\n",
      "Parameters: {'n_estimators': 3, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.045 (std: 0.111)\n",
      "Parameters: {'n_estimators': 25, 'loss': 'square'}\n",
      "\n",
      "RMSE on test data :  1830081.977838165\n",
      "Score with test data -0.007909989520332061\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.122 (std: 0.151)\n",
      "Parameters: {'n_estimators': 9, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.122 (std: 0.085)\n",
      "Parameters: {'n_estimators': 11, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.175 (std: 0.254)\n",
      "Parameters: {'n_estimators': 83, 'loss': 'linear'}\n",
      "\n",
      "RMSE on test data :  7971594.002330143\n",
      "Score with test data -0.05916248045270822\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.068 (std: 0.054)\n",
      "Parameters: {'n_estimators': 3, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.116 (std: 0.102)\n",
      "Parameters: {'n_estimators': 44, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.141 (std: 0.073)\n",
      "Parameters: {'n_estimators': 94, 'loss': 'square'}\n",
      "\n",
      "RMSE on test data :  792721440.6699283\n",
      "Score with test data -0.5256197659829482\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.039 (std: 0.062)\n",
      "Parameters: {'n_estimators': 3, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.101 (std: 0.063)\n",
      "Parameters: {'n_estimators': 5, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.116 (std: 0.128)\n",
      "Parameters: {'n_estimators': 7, 'loss': 'linear'}\n",
      "\n",
      "RMSE on test data :  51790561.785706684\n",
      "Score with test data -0.22511378061348283\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.190 (std: 0.156)\n",
      "Parameters: {'n_estimators': 4, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.085 (std: 0.147)\n",
      "Parameters: {'n_estimators': 8, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.064 (std: 0.193)\n",
      "Parameters: {'n_estimators': 5, 'loss': 'linear'}\n",
      "\n",
      "RMSE on test data :  193670.76477776121\n",
      "Score with test data 0.247456645957936\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.130 (std: 0.095)\n",
      "Parameters: {'n_estimators': 3, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.213 (std: 0.215)\n",
      "Parameters: {'n_estimators': 6, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.246 (std: 0.334)\n",
      "Parameters: {'n_estimators': 1, 'loss': 'exponential'}\n",
      "\n",
      "RMSE on test data :  726.2665645844263\n",
      "Score with test data -6.853319003655169\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.084 (std: 0.075)\n",
      "Parameters: {'n_estimators': 3, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.144 (std: 0.254)\n",
      "Parameters: {'n_estimators': 1, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.198 (std: 0.218)\n",
      "Parameters: {'n_estimators': 19, 'loss': 'square'}\n",
      "\n",
      "RMSE on test data :  47546.665762113684\n",
      "Score with test data -0.06160846435782208\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.083 (std: 0.108)\n",
      "Parameters: {'n_estimators': 20, 'loss': 'exponential'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.177 (std: 0.165)\n",
      "Parameters: {'n_estimators': 48, 'loss': 'linear'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.217 (std: 0.233)\n",
      "Parameters: {'n_estimators': 36, 'loss': 'exponential'}\n",
      "\n",
      "RMSE on test data :  1223.6129167181214\n",
      "Score with test data -0.5666577720654359\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.153 (std: 0.192)\n",
      "Parameters: {'n_estimators': 12, 'loss': 'square'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.547 (std: 0.744)\n",
      "Parameters: {'n_estimators': 17, 'loss': 'linear'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.551 (std: 0.425)\n",
      "Parameters: {'n_estimators': 13, 'loss': 'linear'}\n",
      "\n",
      "RMSE on test data :  131836.05619177755\n",
      "Score with test data -1.2343781414839614\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.099 (std: 0.244)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.099 (std: 0.244)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.099 (std: 0.244)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.099 (std: 0.244)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  393332205.7104095\n",
      "Score with test data 0.008955258101751573\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.269 (std: 0.555)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.269 (std: 0.555)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.269 (std: 0.555)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.269 (std: 0.555)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  1618890917.6681294\n",
      "Score with test data -0.12481976936202764\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.152 (std: 0.130)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.152 (std: 0.130)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.152 (std: 0.130)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.152 (std: 0.130)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  1743126.9037472345\n",
      "Score with test data 0.06877307937130883\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: 0.208 (std: 0.152)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.208 (std: 0.152)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.208 (std: 0.152)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.208 (std: 0.152)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  1658040.601921504\n",
      "Score with test data 0.08684107818979403\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.081 (std: 0.098)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.081 (std: 0.098)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.081 (std: 0.098)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.081 (std: 0.098)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  7179859.149276792\n",
      "Score with test data 0.04603302380600427\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.218 (std: 0.432)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.218 (std: 0.432)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.218 (std: 0.432)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.218 (std: 0.432)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  625585111.2827327\n",
      "Score with test data -0.20396013292009907\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.050 (std: 0.076)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.050 (std: 0.076)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.050 (std: 0.076)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.050 (std: 0.076)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  46333465.481407695\n",
      "Score with test data -0.09602532020646115\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: 0.154 (std: 0.179)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.154 (std: 0.179)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.154 (std: 0.179)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.154 (std: 0.179)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  180986.64221432214\n",
      "Score with test data 0.2967431355730539\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.066 (std: 0.066)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.066 (std: 0.066)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.066 (std: 0.066)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  106.38853391507217\n",
      "Score with test data -0.15040831549821942\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.020 (std: 0.053)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.020 (std: 0.053)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.020 (std: 0.053)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  44754.07487458433\n",
      "Score with test data 0.0007437127332575733\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.004 (std: 0.068)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.004 (std: 0.068)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.004 (std: 0.068)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.004 (std: 0.068)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  696.2762135970926\n",
      "Score with test data 0.10851991946774453\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
      "/home/sid/miniconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:266: UserWarning: The total space of parameters 8 is smaller than n_iter=25. Running 8 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  % (grid_size, self.n_iter, grid_size), UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: -0.020 (std: 0.052)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.020 (std: 0.052)\n",
      "Parameters: {'normalize': False, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.020 (std: 0.052)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.020 (std: 0.052)\n",
      "Parameters: {'normalize': True, 'fit_intercept': True, 'copy_X': False}\n",
      "\n",
      "RMSE on test data :  58679.24818607845\n",
      "Score with test data 0.005494905620035184\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.073 (std: 0.145)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'constant', 'hidden_layer_sizes': 41, 'alpha': 0.4941026728606091, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.074 (std: 0.186)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'constant', 'hidden_layer_sizes': 94, 'alpha': 0.3789090317690659, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.074 (std: 0.151)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 106, 'alpha': 0.9549437438348258, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  0.7353579168966489\n",
      "Score with test data 0.004860464723698832\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.054 (std: 0.077)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 118, 'alpha': 0.44338590906735953, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.079 (std: 0.152)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 62, 'alpha': 0.8276803345873588, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.120 (std: 0.104)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 198, 'alpha': 0.15348565143382523, 'activation': 'tanh'}\n",
      "\n",
      "RMSE on test data :  0.2403991518929062\n",
      "Score with test data -0.11355780277209804\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.191 (std: 0.110)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': True, 'learning_rate': 'constant', 'hidden_layer_sizes': 90, 'alpha': 0.3740250265634144, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.181 (std: 0.138)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 86, 'alpha': 0.932470662048248, 'activation': 'tanh'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.179 (std: 0.103)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': False, 'learning_rate': 'constant', 'hidden_layer_sizes': 24, 'alpha': 0.6232761565847192, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  2.479889408636079\n",
      "Score with test data -0.06935273829624666\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.281 (std: 0.134)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 153, 'alpha': 0.31427687198595927, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.266 (std: 0.142)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 132, 'alpha': 0.5932824214060526, 'activation': 'relu'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.248 (std: 0.152)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 58, 'alpha': 0.3802544170216672, 'activation': 'relu'}\n",
      "\n",
      "RMSE on test data :  3.060646269146579\n",
      "Score with test data 0.002377742037854569\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.105 (std: 0.077)\n",
      "Parameters: {'solver': 'adam', 'shuffle': True, 'learning_rate': 'constant', 'hidden_layer_sizes': 47, 'alpha': 0.21037282226515874, 'activation': 'tanh'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.102 (std: 0.055)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 99, 'alpha': 0.8716843535421861, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.101 (std: 0.081)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 11, 'alpha': 0.6490268857536547, 'activation': 'identity'}\n",
      "\n",
      "RMSE on test data :  2.1241716925494063\n",
      "Score with test data 0.0540768404541474\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.086 (std: 0.157)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 94, 'alpha': 0.03802090526203283, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.089 (std: 0.264)\n",
      "Parameters: {'solver': 'adam', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 4, 'alpha': 0.0852131647541146, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.095 (std: 0.102)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 33, 'alpha': 0.5169575149057389, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  0.1316290531039881\n",
      "Score with test data -0.039577283081243086\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.057 (std: 0.074)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 129, 'alpha': 0.5657565639978531, 'activation': 'tanh'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.056 (std: 0.070)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 135, 'alpha': 0.6862593207262582, 'activation': 'tanh'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.053 (std: 0.113)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 164, 'alpha': 0.3770681771477451, 'activation': 'relu'}\n",
      "\n",
      "RMSE on test data :  0.727130418250949\n",
      "Score with test data -0.06340763687736128\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.280 (std: 0.186)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 54, 'alpha': 0.7578072721418683, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.279 (std: 0.187)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 35, 'alpha': 0.8321512957364223, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.273 (std: 0.181)\n",
      "Parameters: {'solver': 'lbfgs', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 13, 'alpha': 0.8806771904432434, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  0.494837830352222\n",
      "Score with test data 0.23976093966835554\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.018 (std: 0.039)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 157, 'alpha': 0.7657199790856277, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.030 (std: 0.033)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 182, 'alpha': 0.9502719810194105, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.037 (std: 0.076)\n",
      "Parameters: {'solver': 'adam', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 38, 'alpha': 0.04782729361576818, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  0.19208660500026983\n",
      "Score with test data -0.059397593505816415\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -0.004 (std: 0.037)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 89, 'alpha': 0.6714073344817562, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.009 (std: 0.045)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 30, 'alpha': 0.7333776443217689, 'activation': 'identity'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.014 (std: 0.032)\n",
      "Parameters: {'solver': 'adam', 'shuffle': True, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 25, 'alpha': 0.39033341053487464, 'activation': 'tanh'}\n",
      "\n",
      "RMSE on test data :  0.3738328966722449\n",
      "Score with test data 0.04103230091680554\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.018 (std: 0.054)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'constant', 'hidden_layer_sizes': 160, 'alpha': 0.9953047897179185, 'activation': 'tanh'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.017 (std: 0.054)\n",
      "Parameters: {'solver': 'adam', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 157, 'alpha': 0.7293131414238856, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.017 (std: 0.054)\n",
      "Parameters: {'solver': 'sgd', 'shuffle': True, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 165, 'alpha': 0.3333552537784727, 'activation': 'tanh'}\n",
      "\n",
      "RMSE on test data :  0.34356921968849635\n",
      "Score with test data 0.12210435690718091\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: -0.001 (std: 0.034)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'invscaling', 'hidden_layer_sizes': 66, 'alpha': 0.7940114449657545, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: -0.002 (std: 0.035)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 107, 'alpha': 0.6729853037777666, 'activation': 'logistic'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -0.002 (std: 0.036)\n",
      "Parameters: {'solver': 'adam', 'shuffle': False, 'learning_rate': 'adaptive', 'hidden_layer_sizes': 131, 'alpha': 0.43094650040671617, 'activation': 'logistic'}\n",
      "\n",
      "RMSE on test data :  0.35520960127327167\n",
      "Score with test data 0.04092005149213528\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -32.059 (std: 41.846)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -32.059 (std: 41.846)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -32.125 (std: 41.969)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -32.125 (std: 41.969)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  4824068428.75632\n",
      "Score with test data -11.154783110733131\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -44.660 (std: 73.505)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -44.660 (std: 73.505)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -44.683 (std: 73.498)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -44.683 (std: 73.498)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  24361776285.609196\n",
      "Score with test data -15.926778255263368\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -18.271 (std: 4.882)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -18.271 (std: 4.882)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -18.277 (std: 4.899)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -18.277 (std: 4.899)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  7019427.026366254\n",
      "Score with test data -2.7499733383087763\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -19.682 (std: 19.615)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -19.682 (std: 19.615)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -19.684 (std: 19.618)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -19.684 (std: 19.618)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  4546734.57707943\n",
      "Score with test data -1.5040950380535958\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -27.764 (std: 23.267)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -27.764 (std: 23.267)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -27.931 (std: 23.543)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -27.931 (std: 23.543)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  23872615.014827844\n",
      "Score with test data -2.171884835906931\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -53.132 (std: 64.397)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -53.132 (std: 64.397)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -53.142 (std: 64.280)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -53.142 (std: 64.280)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  14070354292.51757\n",
      "Score with test data -26.078882343471015\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -19.001 (std: 7.982)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -19.001 (std: 7.982)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -19.005 (std: 8.050)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -19.005 (std: 8.050)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  1015113501.2812088\n",
      "Score with test data -23.012667490500785\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -14.615 (std: 9.552)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -14.615 (std: 9.552)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -14.623 (std: 9.572)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -14.623 (std: 9.572)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  3649802.6270523723\n",
      "Score with test data -13.18197895642801\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -76.621 (std: 61.021)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -76.621 (std: 61.021)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -76.664 (std: 61.056)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -76.664 (std: 61.056)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  7664.260685412481\n",
      "Score with test data -81.87574703946065\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -40.403 (std: 27.406)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -40.403 (std: 27.406)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -40.405 (std: 27.391)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -40.405 (std: 27.391)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  1297153.2092075862\n",
      "Score with test data -27.962468858562264\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -91.312 (std: 72.738)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -91.312 (std: 72.738)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -91.320 (std: 72.763)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -91.320 (std: 72.763)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  61745.493960122476\n",
      "Score with test data -78.05609419529326\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -44.614 (std: 30.586)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: -44.614 (std: 30.586)\n",
      "Parameters: {'normalize_y': False, 'copy_X_train': False}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -44.625 (std: 30.569)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': True}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: -44.625 (std: 30.569)\n",
      "Parameters: {'normalize_y': True, 'copy_X_train': False}\n",
      "\n",
      "RMSE on test data :  1896775.3300936879\n",
      "Score with test data -31.146845554165537\n",
      "new data\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cl = reg4()\n",
    "cl.start()\n",
    "\n",
    "cl1 = reg5()\n",
    "cl1.start()\n",
    "\n",
    "cl2 = reg9()\n",
    "cl2.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pandas import read_csv\n",
    "import sklearn\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "np.set_printoptions(precision=3, suppress=True) \n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "\n",
    "\n",
    "class Aquatic_toxicity:\n",
    "    \n",
    "    def __init__(self):\n",
    "        return\n",
    "    \n",
    "    def read_data(self):\n",
    "        # reading data \n",
    "        data = np.loadtxt('qsar_aquatic_toxicity.csv', delimiter=';')\n",
    "        return data\n",
    "    \n",
    "    def preprocessing(self, data):               \n",
    "        # splitting data\n",
    "        X = data[:,:-1]\n",
    "        y = data[:,-1]\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)        \n",
    "        #preprocessing using standard scaler\n",
    "        scaler = StandardScaler()\n",
    "        X_train = scaler.fit_transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        return X_train, y_train, X_test, y_test\n",
    "    \n",
    "    def dim_Reduction(self, X_train, X_test):\n",
    "        # Dimensionality Reduction using PCA from 123 dims to 20 dims\n",
    "        pca = PCA(n_components = 20)\n",
    "        X_train = pca.fit_transform(X_train)\n",
    "        X_test = pca.transform(X_test)\n",
    "        return X_train, X_test\n",
    "        \n",
    "    def cv_SVR(self, X, y):\n",
    "        #scorer = make_scorer(neg_mean_squared_error)\n",
    "        C_grid = [0.1, 1, 10]\n",
    "        gamma_grid = np.logspace(-2, 1, 4)[0:3]\n",
    "        svm = sklearn.svm.SVR(kernel='rbf')\n",
    "        param_grid = { 'C' : C_grid, 'gamma' : gamma_grid, 'kernel' : ['rbf', 'sigmoid',  'linear']}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(svm, param_grid, n_jobs=-1, verbose=1, cv=3)\n",
    "        #, scoring = 'neg_mean_squared_error'\n",
    "        gridcv.fit(X_train, y_train)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%% neg mean squared error on validation sets (average)\" % (gridcv.best_score_*100))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_DTR(self, X, y):\n",
    "        dt = DecisionTreeRegressor()\n",
    "        param_grid = {\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "            'criterion' : ['mse', 'mae', 'friedman_mse'],\n",
    "            'splitter' : ['best', 'random'],\n",
    "        }\n",
    "        return Aquatic_toxicity.randomCV(dt, X, y, param_grid, 100, 6)\n",
    "        \n",
    "    def cv_RandomForest(self, X, y):\n",
    "        rf = RandomForestRegressor()\n",
    "        param_grid = {\n",
    "            \"n_estimators\" : [10*x for x in np.arange(1,50)],\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "        }\n",
    "        return Aquatic_toxicity.randomCV(rf, X, y, param_grid, 30, 6)\n",
    "        \n",
    "    def cv_adaBoost(self, X, y):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        ada_boost = AdaBoostRegressor(n_estimators=50, learning_rate=1)\n",
    "        param_grid = {'n_estimators': range(1, 50), 'learning_rate': [0.1, 0.5, 1]}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(ada_boost, param_grid, verbose=1, cv=3, n_jobs=-1)\n",
    "                                                      #, scoring='explained_variance')\n",
    "        gridcv.fit(X, y)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%% validation on validation sets (average)\" % (gridcv.best_score_))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_linReg(self, X, y):\n",
    "        lr = LinearRegression()\n",
    "        param_grid = {\n",
    "            \"fit_intercept\" : [True, False],\n",
    "        }\n",
    "        return Aquatic_toxicity.randomCV(lr, X, y, param_grid, 50, 6)\n",
    "        \n",
    "    def cv_GP(self, X, y):\n",
    "        clf = GaussianProcessRegressor()\n",
    "        param_grid = {\n",
    "            \n",
    "        \"normalize_y\" : [True, False],\n",
    "        \"copy_X_train\" : [True, False],\n",
    "        \"alpha\" : np.linspace(0, 5, 100),\n",
    "        }\n",
    "        return Aquatic_toxicity.randomCV(clf, X, y, param_grid, 25, 6)\n",
    "    \n",
    "    def cv_NNRegressor(self, X, y):\n",
    "        nn = sklearn.neural_network.MLPRegressor(hidden_layer_sizes=(50,),\n",
    "                                           solver='sgd', batch_size=100, max_iter=10,\n",
    "                                           learning_rate_init=.01, momentum=0.9, alpha=0.05,\n",
    "                                           verbose=False, random_state=0)\n",
    "\n",
    "        param_grid ={\n",
    "                    'hidden_layer_sizes' : range(2,100),\n",
    "                    \"activation\" : ['identity', 'logistic', 'tanh', 'relu']\n",
    "                    }\n",
    "        return Aquatic_toxicity.randomCV(nn, X, y, param_grid, 100, 6)\n",
    "        \n",
    "    def randomCV(clf, X, y, param_grid, n_iter, cv):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid, n_iter = n_iter, cv = cv, iid = False)\n",
    "        #scoring = \"explained_variance\"\n",
    "        random_search.fit(X, y)\n",
    "        #print(random_search.cv_results_)\n",
    "        Aquatic_toxicity.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "    \n",
    "    def report(results, n_top=1):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            k = 0\n",
    "            for candidate in candidates:                \n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Variance on validation data: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                k += 1\n",
    "                if k == 3:\n",
    "                    break\n",
    "                \n",
    "    def predict(self, model, X_test, y_test):\n",
    "        predict = model.predict(X_test)\n",
    "        predict[predict<0] =0\n",
    "        rmse = mean_squared_error(y_test, predict)\n",
    "        print(\"MSE on test data : \", rmse)\n",
    "        \n",
    "if __name__ == \"__main__\":\n",
    "    obj = Aquatic_toxicity()\n",
    "    data = obj.read_data()\n",
    "    X_train, y_train, X_test, y_test = obj.preprocessing(data)\n",
    "    # X_train, X_test = obj.dim_Reduction(X_train, X_test)\n",
    "    print('---------SVR--------')\n",
    "    model = obj.cv_SVR(X_train, y_train)\n",
    "    reg = sklearn.svm.SVR().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------DTR--------')\n",
    "    model = obj.cv_DTR(X_train, y_train)\n",
    "    reg = sklearn.tree.DecisionTreeRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Random Forrest Regressor--------')\n",
    "    # taking more than 3 mins \n",
    "    model = obj.cv_RandomForest(X_train, y_train)\n",
    "    reg = sklearn.ensemble.RandomForestRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Adaboost Regressor--------')\n",
    "    model = obj.cv_adaBoost(X_train, y_train)\n",
    "    reg = sklearn.ensemble.AdaBoostRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Gaussian Process Regressor--------')\n",
    "    model = obj.cv_GP(X_train, y_train)\n",
    "    reg = sklearn.gaussian_process.GaussianProcessRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Linear Regressor--------')\n",
    "    model = obj.cv_linReg(X_train, y_train)\n",
    "    reg = LinearRegression().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------NN Regressor--------')\n",
    "    model = obj.cv_NNRegressor(X_train, y_train)\n",
    "    reg = MLPRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pandas import read_csv\n",
    "import sklearn\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "np.set_printoptions(precision=3, suppress=True) \n",
    "from datetime import datetime\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "\n",
    "class Bike_Sharing:\n",
    "    \n",
    "    def __init__(self):\n",
    "        return\n",
    "    \n",
    "    # reading data \n",
    "    def read_data(self):        \n",
    "       #Dropping not predictive attributes : instant\n",
    "        data = np.loadtxt('hour.csv', delimiter=',', skiprows=1, usecols=(range(2,17)))\n",
    "        data2 = np.loadtxt('hour.csv', delimiter=',', skiprows=1, usecols=(1), dtype='|S10').astype(str)\n",
    "        \n",
    "        # Preprocessing step: extracting week number from date attribute\n",
    "        for i in range(data2.shape[0]):\n",
    "            data2[i] = datetime.date(datetime.strptime(data2[i], '%Y-%m-%d')).isocalendar()[1]\n",
    "        data2 = data2.astype(int)        \n",
    "        data = np.column_stack((data2.reshape(-1,1), data))\n",
    "        return data\n",
    "    \n",
    "    def preprocessing(self, data):               \n",
    "        # splitting data\n",
    "        X = data[:,:-1]\n",
    "        y = data[:,-1]\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)        \n",
    "        #preprocessing using standard scaler\n",
    "        scaler = StandardScaler()\n",
    "        X_train = scaler.fit_transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        return X_train, y_train, X_test, y_test\n",
    "    \n",
    "        \n",
    "    def cv_SVR(self, X, y):\n",
    "        #scorer = make_scorer(neg_mean_squared_error)\n",
    "        C_grid = [0.1, 1, 10]\n",
    "        gamma_grid = np.logspace(-2, 1, 4)[0:3]\n",
    "        svm = sklearn.svm.SVR(kernel='rbf')\n",
    "        param_grid = { 'C' : C_grid, 'gamma' : gamma_grid, 'kernel' : ['rbf', 'sigmoid',  'linear']}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(svm, param_grid, n_jobs=-1, verbose=1, cv=3)\n",
    "        #, scoring = 'neg_mean_squared_error'\n",
    "        gridcv.fit(X_train, y_train)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%% neg mean squared error on validation sets (average)\" % (gridcv.best_score_*100))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_DTR(self, X, y):\n",
    "        dt = DecisionTreeRegressor()\n",
    "        param_grid = {\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "            'criterion' : ['mse', 'mae', 'friedman_mse'],\n",
    "            'splitter' : ['best', 'random'],\n",
    "        }\n",
    "        return Bike_Sharing.randomCV(dt, X, y, param_grid, 50, 6)\n",
    "        \n",
    "    def cv_RandomForest(self, X, y):\n",
    "        rf = RandomForestRegressor()\n",
    "        param_grid = {\n",
    "            #\"n_estimators\" : [10*x for x in np.arange(1,25)],\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "        }\n",
    "        return Bike_Sharing.randomCV(rf, X, y, param_grid, 40, 6)\n",
    "        \n",
    "    def cv_adaBoost(self, X, y):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        ada_boost = AdaBoostRegressor(n_estimators=50, learning_rate=1)\n",
    "        param_grid = {'n_estimators': range(1, 50), 'learning_rate': [0.1, 0.5, 1]}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(ada_boost, param_grid, verbose=1, cv=3, n_jobs=-1)\n",
    "                                                      #, scoring='explained_variance')\n",
    "        gridcv.fit(X, y)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%% validation on validation sets (average)\" % (gridcv.best_score_))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_linReg(self, X, y):\n",
    "        lr = LinearRegression()\n",
    "        param_grid = {\n",
    "            \"fit_intercept\" : [True, False],\n",
    "        }\n",
    "        return Bike_Sharing.randomCV(lr, X, y, param_grid, 40, 6)\n",
    "        \n",
    "    def cv_GP(self, X, y):\n",
    "        clf = GaussianProcessRegressor()\n",
    "        param_grid = {\n",
    "            \n",
    "        \"normalize_y\" : [True, False],\n",
    "        \"copy_X_train\" : [True, False],\n",
    "        \"alpha\" : np.linspace(0, 5, 100),\n",
    "        }\n",
    "        return Bike_Sharing.randomCV(clf, X, y, param_grid, 10, 6)\n",
    "    \n",
    "    def cv_NNRegressor(self, X, y):\n",
    "        nn = sklearn.neural_network.MLPRegressor()\n",
    "\n",
    "        param_grid ={\n",
    "                    'hidden_layer_sizes' : range(2,100),\n",
    "                    \"activation\" : ['identity', 'logistic', 'tanh', 'relu']\n",
    "                    }\n",
    "        return Bike_Sharing.randomCV(nn, X, y, param_grid, 50, 5)\n",
    "        \n",
    "    def randomCV(clf, X, y, param_grid, n_iter, cv):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid, n_iter = n_iter, cv = cv, iid = False, \n",
    "                                           verbose=1, n_jobs=-1, scoring='explained_variance')\n",
    "        #scoring = \"explained_variance\"\n",
    "        random_search.fit(X, y)\n",
    "        #print(random_search.cv_results_)\n",
    "        Bike_Sharing.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "    \n",
    "    def report(results, n_top=1):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            k = 0\n",
    "            for candidate in candidates:                \n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Variance on validation data: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                k += 1\n",
    "                if k == 3:\n",
    "                    break\n",
    "                \n",
    "    def predict(self, model, X_test, y_test):\n",
    "        predict = model.predict(X_test)\n",
    "        predict[predict<0] =0\n",
    "        rmse = mean_squared_error(y_test, predict)\n",
    "        print(\"MSE on test data : \", rmse)\n",
    "        \n",
    "if __name__ == \"__main__\":\n",
    "    obj = Bike_Sharing()\n",
    "    data = obj.read_data()\n",
    "    X_train, y_train, X_test, y_test = obj.preprocessing(data)\n",
    "    print('---------SVR--------')\n",
    "    model = obj.cv_SVR(X_train, y_train)\n",
    "    reg = sklearn.svm.SVR().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------DTR--------')\n",
    "    model = obj.cv_DTR(X_train, y_train)\n",
    "    reg = sklearn.tree.DecisionTreeRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Random Forrest Regressor--------') \n",
    "    model = obj.cv_RandomForest(X_train, y_train)\n",
    "    reg = sklearn.ensemble.RandomForestRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Adaboost Regressor--------')\n",
    "    model = obj.cv_adaBoost(X_train, y_train)\n",
    "    reg = sklearn.ensemble.AdaBoostRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Gaussian Process Regressor--------')\n",
    "    model = obj.cv_GP(X_train, y_train)\n",
    "    reg = sklearn.gaussian_process.GaussianProcessRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Linear Regressor--------')\n",
    "    model = obj.cv_linReg(X_train, y_train)\n",
    "    reg = LinearRegression().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------NN Regressor--------')\n",
    "    model = obj.cv_NNRegressor(X_train, y_train)\n",
    "    reg = MLPRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "compresive_strength_concrete.csv not found.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-086ea83bc314>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    152\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"__main__\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m     \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mConcrete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 154\u001b[0;31m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    155\u001b[0m     \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m     \u001b[0;31m#print(X_train)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-086ea83bc314>\u001b[0m in \u001b[0;36mread_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m#Dropping not predictive attributes : instant\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadtxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'compresive_strength_concrete.csv'\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelimiter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36mloadtxt\u001b[0;34m(fname, dtype, comments, delimiter, converters, skiprows, usecols, unpack, ndmin, encoding, max_rows)\u001b[0m\n\u001b[1;32m    960\u001b[0m             \u001b[0mfname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos_fspath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_is_string_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 962\u001b[0;31m             \u001b[0mfh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_datasource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    963\u001b[0m             \u001b[0mfencoding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'encoding'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'latin1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m             \u001b[0mfh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/numpy/lib/_datasource.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(path, mode, destpath, encoding, newline)\u001b[0m\n\u001b[1;32m    264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m     \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataSource\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdestpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnewline\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnewline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/numpy/lib/_datasource.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, mode, encoding, newline)\u001b[0m\n\u001b[1;32m    622\u001b[0m                                       encoding=encoding, newline=newline)\n\u001b[1;32m    623\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 624\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s not found.\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: compresive_strength_concrete.csv not found."
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from pandas import read_csv\n",
    "import sklearn\n",
    "from io import StringIO\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "np.set_printoptions(precision=3, suppress=True) \n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning) \n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "\n",
    "class Concrete:\n",
    "    \n",
    "    def __init__(self):\n",
    "        return\n",
    "    \n",
    "    # reading data \n",
    "    def read_data(self):        \n",
    "        #Dropping not predictive attributes : instant\n",
    "        data = np.loadtxt('compresive_strength_concrete.csv',  skiprows=1, delimiter=',')\n",
    "        return data\n",
    "    \n",
    "    def preprocessing(self, data):               \n",
    "        # splitting data\n",
    "        X = data[:,:-1]\n",
    "        y = data[:,-1]\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)        \n",
    "        #preprocessing using standard scaler\n",
    "        scaler = StandardScaler()\n",
    "        X_train = scaler.fit_transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        return X_train, y_train, X_test, y_test\n",
    "        \n",
    "    def cv_SVR(self, X, y):\n",
    "        #scorer = make_scorer(neg_mean_squared_error)\n",
    "        C_grid = [0.1, 1, 10]\n",
    "        gamma_grid = np.logspace(-2, 1, 4)[0:3]\n",
    "        svm = sklearn.svm.SVR(kernel='rbf')\n",
    "        param_grid = { 'C' : C_grid, 'gamma' : gamma_grid, 'kernel' : ['rbf', 'sigmoid',  'linear']}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(svm, param_grid, n_jobs=-1, verbose=1, cv=3)\n",
    "        #, scoring = 'neg_mean_squared_error'\n",
    "        gridcv.fit(X_train, y_train)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%%  on validation sets (average)\" % (gridcv.best_score_*100))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_DTR(self, X, y):\n",
    "        dt = DecisionTreeRegressor()\n",
    "        param_grid = {\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "            'criterion' : ['mse', 'mae', 'friedman_mse'],\n",
    "            'splitter' : ['best', 'random'],\n",
    "        }\n",
    "        return Concrete.randomCV(dt, X, y, param_grid, 50, 6)\n",
    "        \n",
    "    def cv_RandomForest(self, X, y):\n",
    "        rf = RandomForestRegressor()\n",
    "        param_grid = {\n",
    "            #\"n_estimators\" : [10*x for x in np.arange(1,25)],\n",
    "            \"min_samples_split\" : np.random.random_sample((100,)),\n",
    "            \"min_samples_leaf\" : np.arange(1,6),\n",
    "            'max_depth': range(1, 20),\n",
    "        }\n",
    "        return Concrete.randomCV(rf, X, y, param_grid, 40, 6)\n",
    "    \n",
    "    def cv_GP(self, X, y):\n",
    "        clf = GaussianProcessRegressor()\n",
    "        param_grid = {\n",
    "            \n",
    "        \"normalize_y\" : [True, False],\n",
    "        \"copy_X_train\" : [True, False],\n",
    "        \"alpha\" : np.linspace(0, 5, 100),\n",
    "        }\n",
    "        return Concrete.randomCV(clf, X, y, param_grid, 25, 6)\n",
    "        \n",
    "    def cv_adaBoost(self, X, y):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        ada_boost = AdaBoostRegressor(n_estimators=50, learning_rate=1)\n",
    "        param_grid = {'n_estimators': range(1, 50), 'learning_rate': [0.1, 0.5, 1]}\n",
    "        gridcv = sklearn.model_selection.GridSearchCV(ada_boost, param_grid, verbose=1, cv=3, n_jobs=-1)\n",
    "                                                      #, scoring='explained_variance')\n",
    "        gridcv.fit(X, y)\n",
    "        print(\"best parameters:\", gridcv.best_params_)\n",
    "        print(\"%.1f%% validation on validation sets (average)\" % (gridcv.best_score_))\n",
    "        return gridcv.best_params_\n",
    "    \n",
    "    def cv_linReg(self, X, y):\n",
    "        lr = LinearRegression()\n",
    "        param_grid = {\n",
    "            \"fit_intercept\" : [True, False],\n",
    "        }\n",
    "        return Concrete.randomCV(lr, X, y, param_grid, 40, 6)\n",
    "        \n",
    "   \n",
    "    \n",
    "    def cv_NNRegressor(self, X, y):\n",
    "        nn = sklearn.neural_network.MLPRegressor()\n",
    "\n",
    "        param_grid ={\n",
    "                    'hidden_layer_sizes' : range(2,100),\n",
    "                    \"activation\" : ['identity', 'logistic', 'tanh', 'relu']\n",
    "                    }\n",
    "        return Concrete.randomCV(nn, X, y, param_grid, 100, 6)\n",
    "        \n",
    "    def randomCV(clf, X, y, param_grid, n_iter, cv):\n",
    "        #scorer = make_scorer(precision_score)\n",
    "        random_search = RandomizedSearchCV(clf, param_distributions = param_grid, n_iter = n_iter, cv = cv, iid = False, \n",
    "                                           verbose=1, n_jobs=-1)\n",
    "        #scoring = \"explained_variance\"\n",
    "        random_search.fit(X, y)\n",
    "        #print(random_search.cv_results_)\n",
    "        Concrete.report(random_search.cv_results_)\n",
    "        return random_search.best_params_\n",
    "    \n",
    "    def report(results, n_top=1):\n",
    "        for i in range(1, n_top + 1):\n",
    "            candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "            k = 0\n",
    "            for candidate in candidates:                \n",
    "                print(\"Model with rank: {0}\".format(i))\n",
    "                print(\"Variance on validation data: {0:.3f} (std: {1:.3f})\".format(\n",
    "                      results['mean_test_score'][candidate],\n",
    "                      results['std_test_score'][candidate]))\n",
    "                print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "                print(\"\")\n",
    "                k += 1\n",
    "                if k == 3:\n",
    "                    break\n",
    "                \n",
    "    def predict(self, model, X_test, y_test):\n",
    "        predict = model.predict(X_test)\n",
    "        predict[predict<0] =0\n",
    "        rmse = mean_squared_error(y_test, predict)\n",
    "        print(\"MSE on test data : \", rmse)\n",
    "        \n",
    "if __name__ == \"__main__\":\n",
    "    obj = Concrete()\n",
    "    data = obj.read_data()\n",
    "    X_train, y_train, X_test, y_test = obj.preprocessing(data)\n",
    "    #print(X_train)\n",
    "    #print(y_train)\n",
    "    print('---------SVR--------')\n",
    "    model = obj.cv_SVR(X_train, y_train)\n",
    "    reg = sklearn.svm.SVR().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------DTR--------')\n",
    "    model = obj.cv_DTR(X_train, y_train)\n",
    "    reg = sklearn.tree.DecisionTreeRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Random Forrest Regressor--------') \n",
    "    model = obj.cv_RandomForest(X_train, y_train)\n",
    "    reg = sklearn.ensemble.RandomForestRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Adaboost Regressor--------')\n",
    "    model = obj.cv_adaBoost(X_train, y_train)\n",
    "    reg = sklearn.ensemble.AdaBoostRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Gaussian Process Regressor--------')\n",
    "    model = obj.cv_GP(X_train, y_train)\n",
    "    reg = sklearn.gaussian_process.GaussianProcessRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------Linear Regressor--------')\n",
    "    model = obj.cv_linReg(X_train, y_train)\n",
    "    reg = LinearRegression().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)\n",
    "    print('---------NN Regressor--------')\n",
    "    model = obj.cv_NNRegressor(X_train, y_train)\n",
    "    reg = MLPRegressor().set_params(**model).fit(X_train, y_train)\n",
    "    obj.predict(reg, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sklearn\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import preprocessing\n",
    "from io import StringIO\n",
    "import scipy\n",
    "import scipy.stats               # For reciprocal distribution\n",
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn import svm\n",
    "import sklearn.gaussian_process \n",
    "import sklearn.tree        # For DecisionTreeClassifier class\n",
    "import sklearn.ensemble    # For RandomForestClassifier class\n",
    "import sklearn.linear_model # For Logistic Classifier\n",
    "#from sklearn.neighbors import LSHForest\n",
    "import sklearn.naive_bayes #For Naive Bayes\n",
    "import sklearn.neural_network #For MLP classifier\n",
    "import csv\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # Ignore sklearn deprecation warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)       # Ignore sklearn deprecation warnings\n",
    "np.set_printoptions(precision=20, suppress=True)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class student_performance:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def preprocess_data(self,file):\n",
    "        f = open(file,\"r\")\n",
    "        c = StringIO(f.read())\n",
    "        print(\"READING TRAIN DATA\")\n",
    "       \n",
    "        #Read data for One hot encoding\n",
    "        X_string_OH = np.char.strip(np.genfromtxt(c,dtype='str',delimiter = ';',usecols = (0,1,3,5,8,9,10,11),skip_header=1))\n",
    "        f = open(file,\"r\")\n",
    "        c = StringIO(f.read())\n",
    "        \n",
    "        #Read Data for ordinal encoding\n",
    "        X_string_OE = np.char.strip(np.genfromtxt(c,dtype='str',delimiter = ';',usecols = (4,15,16,17,18,19,20,21,22),skip_header=1))\n",
    "        X_float = np.loadtxt(file,delimiter = \";\",usecols = (2,6,7,12,13,14,23,24,25,26,27,28,29,30,31,32), dtype = np.float,skiprows=1).astype(int)\n",
    "        print(X_string_OE.shape)\n",
    "        print(X_string_OH.shape)\n",
    "        print(X_float.shape)\n",
    "        X_soh = self.preprocess_data_OH(X_string_OH)\n",
    "        X_soe = self.preprocess_data_OE(X_string_OE)\n",
    "        X = np.column_stack((X_soh,X_soe,X_float[:,:-1]))\n",
    "        y = X_float[:,-1]\n",
    "        return(X,y)\n",
    "    \n",
    "    def preprocess_data_OH(self,X):\n",
    "        X=X.tolist()\n",
    "        encoder = preprocessing.OneHotEncoder(sparse = False)\n",
    "        print(encoder.fit(X))\n",
    "        X = encoder.transform(X)\n",
    "        print(X[0])\n",
    "        return(X)\n",
    "    \n",
    "    def preprocess_data_OE(self,X):\n",
    "        X=X.tolist()\n",
    "        encoder = preprocessing.OrdinalEncoder()\n",
    "        encoder.fit(X)\n",
    "        X = encoder.transform(X)\n",
    "        print(X[5])\n",
    "        return(X)\n",
    "    \n",
    "    def scale_data_1(self,X):\n",
    "        scaler = preprocessing.MinMaxScaler((0,1)).fit(X)\n",
    "        return(scaler)\n",
    "    \n",
    "    def scale_data_5(self,X):\n",
    " \n",
    "        scaler = preprocessing.MinMaxScaler((0,5)).fit(X)\n",
    "        return(scaler)\n",
    "    \n",
    "    def random_CV(self,clf,X,y,param_grid,n_iter,cv):\n",
    " \n",
    "        print(\"Starting search\")\n",
    "       \n",
    "        random_search = model_selection.RandomizedSearchCV(clf, param_distributions = param_grid,n_iter = n_iter, cv = cv,\n",
    "                                           iid = False,verbose=1,n_jobs = 4)\n",
    "        random_search.fit(X, y)\n",
    "        print(\"best parameters:\", random_search.best_params_)\n",
    "        print(\"%.1f%% accuracy on validation sets (average)\" % (random_search.best_score_*100))\n",
    "        return random_search.best_params_\n",
    "    \n",
    "    def SVR_reg(self,X,y):\n",
    "        print(\"SVR classifier called\")\n",
    "        svr_clf = svm.SVR()\n",
    "        param_dist = {\n",
    "            'C'     : scipy.stats.reciprocal(1.0, 1000.),\n",
    "            'kernel': ['rbf','poly','linear'],\n",
    "            'degree': [2],\n",
    "            'gamma' : scipy.stats.reciprocal(0.01, 10.)\n",
    "        }\n",
    "        return self.random_CV(svr_clf,X,y,param_dist,15,3)\n",
    "    \n",
    "    def DTR_reg(self,X,y):\n",
    "        print(\"Decision Tree Regressor called\")\n",
    "        dt_reg = sklearn.tree.DecisionTreeRegressor()\n",
    "        param_dist  ={\n",
    "            \"splitter\" : ['best', 'random'],\n",
    "            \"max_depth\" :[None,100,150,200,250,300],\n",
    "            \"max_features\": [\"sqrt\",\"log2\",None]\n",
    "        }\n",
    "        return self.random_CV(dt_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def RF_reg(self,X,y):\n",
    "        print(\"Random Forest Regressor Called\")\n",
    "        rf_reg = sklearn.ensemble.RandomForestRegressor()\n",
    "        param_dist = {\n",
    "            \"n_estimators\" : [10,25,50,75,100,125,150,175,200],\n",
    "            \"max_depth\" :[None,100,150,200,250,300],\n",
    "            \"max_features\": [\"sqrt\",\"log2\",None]\n",
    "        }\n",
    "        return self.random_CV(rf_reg,X,y,param_dist,12,3)\n",
    "    \n",
    "    def ADB_reg(self,X,y):\n",
    "        print(\"Ada Boost Regressor called\")\n",
    "        adb_reg = sklearn.ensemble.AdaBoostRegressor()\n",
    "        param_dist = {\n",
    "            \"n_estimators\" : [50,75,100,150,200,250,300,400,500],\n",
    "            \"loss\" : ['linear', 'square', 'exponential'],\n",
    "            \"learning_rate\" : [0.5,0.75,1,1.5]\n",
    "        }\n",
    "        return self.random_CV(adb_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def GP_reg (self,X,y):\n",
    "        print(\"Gaussian Process Regression called\")\n",
    "        \n",
    "        gp_reg = sklearn.gaussian_process.GaussianProcessRegressor()\n",
    "        param_dist = {\n",
    "            'alpha' : scipy.stats.reciprocal(np.exp(-12), np.exp(-1)),\n",
    "            \"normalize_y\" : [False, True]\n",
    "        }\n",
    "        return self.random_CV(gp_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def LR_reg(self,X,y):\n",
    "        print(\"Linear Regression called\")\n",
    "        \n",
    "        lr_reg = sklearn.linear_model.LinearRegression()\n",
    "        param_dist = {\n",
    "            'fit_intercept' : [True, False],\n",
    "            'normalize': [True,False]\n",
    "        }\n",
    "        return self.random_CV(lr_reg,X,y,param_dist,4,3)\n",
    "        \n",
    "    def MLP_reg(self,X,y):\n",
    "        print(\"Neural Network regressor Called\")\n",
    "        mlp_reg = sklearn.neural_network.MLPRegressor()\n",
    "        param_dist = {\n",
    "            \"hidden_layer_sizes\" : [(50,), (100,),(50,50),(100,50),(100,100),(50,50,50)],\n",
    "            \"activation\" :['tanh', 'relu','logistic','identity'],\n",
    "            \"solver\" : ['lbfgs', 'sgd'], #smaller Data set, no adam\n",
    "            \"alpha\" : scipy.stats.reciprocal(0.00005,0.1),\n",
    "            \"learning_rate\" : ['constant','invscaling'],\n",
    "            \"max_iter\" : [100,200,300,500,700]\n",
    "        }\n",
    "        return self.random_CV(mlp_reg,X,y,param_dist,15,3)\n",
    "    \n",
    "    def train_reg(self,reg,params,X,y):\n",
    "        reg.set_params(**params)\n",
    "        reg.fit(X,y)\n",
    "        print(\"Complete Training Accuracy\")\n",
    "        print(reg.score(X,y))\n",
    "        return reg\n",
    "    \n",
    "    def start(self):\n",
    "        print(\"******Regression of Student Performance Data Set Begins ******\")\n",
    "        file = 'student-por.csv'\n",
    "        (X,y)=self.preprocess_data(file)\n",
    "        \n",
    "        \n",
    "        X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2)\n",
    "        \n",
    "        print(\"Calculating Standard Scaler using training data\")\n",
    "        scaler_1 = self.scale_data_1(X_train[:,:-2])  #Scale all the first 30 attributes to get a val range of 0,1\n",
    "        scaler_2 = self.scale_data_5(X_train[:,X_train.shape[1]-2:X_train.shape[1]]) #Scale the imp last 2 attributes for a val range 0,5, to show its importance\n",
    "        \n",
    "        X_train_1 = scaler_1.transform(X_train[:,:-2])\n",
    "        X_test_1 = scaler_1.transform(X_test[:,:-2])\n",
    "        \n",
    "        X_train_2 = scaler_2.transform(X_train[:,X_train.shape[1]-2:X_train.shape[1]])\n",
    "        X_test_2 = scaler_2.transform(X_test[:,X_train.shape[1]-2:X_train.shape[1]])\n",
    "\n",
    "        X_train = np.column_stack((X_train_1,X_train_2))\n",
    "        X_test = np.column_stack((X_test_1,X_test_2))\n",
    "        \n",
    "        print(\"**Training of Student_performance starts**\")\n",
    "        \n",
    "        print(\"__Training SVR__\")\n",
    "        \n",
    "        svr_reg = self.train_reg(svm.SVR(),self.SVR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training DTR__\")\n",
    "        dt_reg = self.train_reg(sklearn.tree.DecisionTreeRegressor(),self.DTR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Random Forest Reg__\")\n",
    "        rf_reg = self.train_reg(sklearn.ensemble.RandomForestRegressor(),self.RF_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training AdaBoost Reg__\")\n",
    "        adb_reg = self.train_reg(sklearn.ensemble.AdaBoostRegressor(),self.ADB_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Gaussian proc Reg__\")\n",
    "        gp_reg = self.train_reg(sklearn.gaussian_process.GaussianProcessRegressor(),self.GP_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Linear Reg__\")\n",
    "        lin_reg = self.train_reg(sklearn.linear_model.LinearRegression(),self.LR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Neural Net Reg__\")\n",
    "        nn_reg = self.train_reg(sklearn.neural_network.MLPRegressor(),self.MLP_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"**Test Data Prediction Begins*\")\n",
    "        \n",
    "        \n",
    "        print(\"Testing SVM Regressionr\")\n",
    "        print(svr_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Decision Trees\")\n",
    "        print(dt_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Random Forests\")\n",
    "        print(rf_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Adaboost\")\n",
    "        print(adb_reg.score(X_test,y_test))\n",
    "        \n",
    "        \n",
    "        print(\"Testing Gaussian Process reg\")\n",
    "        print(gp_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Linear Regression\")\n",
    "        print(lin_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Neural Network/MLP\")\n",
    "        print(nn_reg.score(X_test,y_test))\n",
    "        \n",
    "        return\n",
    "    \n",
    "obj = student_performance()\n",
    "obj.start()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy as np\n",
    "import sklearn\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import preprocessing\n",
    "from io import StringIO\n",
    "import scipy\n",
    "import scipy.stats               # For reciprocal distribution\n",
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn import svm\n",
    "import sklearn.gaussian_process \n",
    "import sklearn.tree        # For DecisionTreeClassifier class\n",
    "import sklearn.ensemble    # For RandomForestClassifier class\n",
    "import sklearn.linear_model # For Logistic Classifier\n",
    "#from sklearn.neighbors import LSHForest\n",
    "import sklearn.naive_bayes #For Naive Bayes\n",
    "import sklearn.neural_network #For MLP classifier\n",
    "import csv\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # Ignore sklearn deprecation warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)       # Ignore sklearn deprecation warnings\n",
    "np.set_printoptions(precision=20, suppress=True)\n",
    "\n",
    "class wine:\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def SFS(self,X,y): #Feature selection as some features are useless\n",
    "        size = 11\n",
    "        cols = [0,1,2,3,4,5,6,7,8,9,10]\n",
    "        min_score = 0\n",
    "        useless_col = 20\n",
    "        useless_cols = []\n",
    "        rf_reg = sklearn.ensemble.RandomForestRegressor()\n",
    "        while(len(cols)> 8):\n",
    "            min_score = 0\n",
    "            X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)\n",
    "            for i in cols:\n",
    "                if i  in useless_cols:\n",
    "                    continue   \n",
    "                X_sfs = np.column_stack((X_train[:,0:i],X_train[:,(i+1):len(cols)]))\n",
    "                    \n",
    "                rf_reg.fit(X_sfs,y_train);\n",
    "                score = rf_reg.score(np.column_stack((X_test[:,0:i],X_test[:,(i+1):len(cols)])),y_test)\n",
    "                if(score > min_score):\n",
    "                    min_score = score\n",
    "                    print(score)\n",
    "                    print(\"removing %d\" % i)\n",
    "                    useless_col = i\n",
    "            useless_cols.append(useless_col)\n",
    "            #cols.pop()\n",
    "            print(\"deleting\")\n",
    "            print(useless_col)\n",
    "            np.delete(X,useless_col,1)\n",
    "        return useless_cols\n",
    "\n",
    "    \n",
    "    def scale_data(self,X):\n",
    "        scaler = preprocessing.StandardScaler().fit(X)\n",
    "        X = scaler.transform(X)\n",
    "        return(X,scaler)\n",
    "    def random_CV(self,clf,X,y,param_grid,n_iter,cv):\n",
    " \n",
    "        print(\"Starting search\")\n",
    "       \n",
    "        random_search = model_selection.RandomizedSearchCV(clf, param_distributions = param_grid,n_iter = n_iter, cv = cv,\n",
    "                                           iid = False,verbose=1,n_jobs = 4)\n",
    "        random_search.fit(X, y)\n",
    "        print(\"best parameters:\", random_search.best_params_)\n",
    "        print(\"%.1f%% accuracy on validation sets (average)\" % (random_search.best_score_*100))\n",
    "        return random_search.best_params_\n",
    "    \n",
    "    def SVR_reg(self,X,y):\n",
    "        print(\"SVR classifier called\")\n",
    "        svr_clf = svm.SVR()\n",
    "        param_dist = {\n",
    "            'C'     : scipy.stats.reciprocal(1.0, 1000.),\n",
    "            'kernel': ['rbf','linear'],\n",
    "            'degree': [2],\n",
    "            'gamma' : scipy.stats.reciprocal(0.01, 10.)\n",
    "        }\n",
    "        return self.random_CV(svr_clf,X,y,param_dist,5,3)\n",
    "    \n",
    "    def DTR_reg(self,X,y):\n",
    "        print(\"Decision Tree Regressor called\")\n",
    "        dt_reg = sklearn.tree.DecisionTreeRegressor()\n",
    "        param_dist  ={\n",
    "            \"splitter\" : ['best', 'random'],\n",
    "            \"max_depth\" :[None,100,150,200,250,300],\n",
    "            \"max_features\": [\"sqrt\",\"log2\",None]\n",
    "        }\n",
    "        return self.random_CV(dt_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def RF_reg(self,X,y):\n",
    "        print(\"Random Forest Regressor Called\")\n",
    "        rf_reg = sklearn.ensemble.RandomForestRegressor()\n",
    "        param_dist = {\n",
    "            \"n_estimators\" : [10,25,50,75,100,125,150,175,200],\n",
    "            \"max_depth\" :[None,100,150,200,250,300],\n",
    "            \"max_features\": [\"sqrt\",\"log2\",None]\n",
    "        }\n",
    "        return self.random_CV(rf_reg,X,y,param_dist,12,3)\n",
    "    \n",
    "    def ADB_reg(self,X,y):\n",
    "        print(\"Ada Boost Regressor called\")\n",
    "        adb_reg = sklearn.ensemble.AdaBoostRegressor()\n",
    "        param_dist = {\n",
    "            \"n_estimators\" : [50,75,100,150,200,250,300,400,500],\n",
    "            \"loss\" : ['linear', 'square', 'exponential'],\n",
    "            \"learning_rate\" : [0.5,0.75,1,1.5]\n",
    "        }\n",
    "        return self.random_CV(adb_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def GP_reg (self,X,y):\n",
    "        print(\"Gaussian Process Regression called\")\n",
    "        \n",
    "        gp_reg = sklearn.gaussian_process.GaussianProcessRegressor()\n",
    "        param_dist = {\n",
    "            'alpha' : scipy.stats.reciprocal(np.exp(-12), np.exp(-1)),\n",
    "            \"normalize_y\" : [False, True]\n",
    "        }\n",
    "        return self.random_CV(gp_reg,X,y,param_dist,10,3)\n",
    "    \n",
    "    def LR_reg(self,X,y):\n",
    "        print(\"Linear Regression called\")\n",
    "        \n",
    "        lr_reg = sklearn.linear_model.LinearRegression()\n",
    "        param_dist = {\n",
    "            'fit_intercept' : [True, False],\n",
    "            'normalize': [True,False]\n",
    "        }\n",
    "        return self.random_CV(lr_reg,X,y,param_dist,4,3)\n",
    "        \n",
    "    def MLP_reg(self,X,y):\n",
    "        print(\"Neural Network regressor Called\")\n",
    "        mlp_reg = sklearn.neural_network.MLPRegressor()\n",
    "        param_dist = {\n",
    "            \"hidden_layer_sizes\" : [(50,), (100,),(50,50),(100,50),(100,100),(50,50,50)],\n",
    "            \"activation\" :['tanh', 'relu','logistic','identity'],\n",
    "            \"solver\" : ['lbfgs', 'sgd'], #smaller Data set, no adam\n",
    "            \"alpha\" : scipy.stats.reciprocal(0.00005,0.1),\n",
    "            \"learning_rate\" : ['constant','invscaling'],\n",
    "            \"max_iter\" : [100,200,300,500,700]\n",
    "        }\n",
    "        return self.random_CV(mlp_reg,X,y,param_dist,15,3)\n",
    "    \n",
    "    def train_reg(self,reg,params,X,y):\n",
    "        reg.set_params(**params)\n",
    "        reg.fit(X,y)\n",
    "        print(\"Complete Training Accuracy\")\n",
    "        print(reg.score(X,y))\n",
    "        return reg  \n",
    "    \n",
    "    def start(self):\n",
    "        file = 'winequality-white.csv'\n",
    "        \n",
    "        X_float = np.loadtxt(file,delimiter = \";\", dtype = np.float,skiprows=1)\n",
    "        \n",
    "        print(X_float[0])\n",
    "        \n",
    "        X,y = X_float[:,:-1],X_float[:,-1]\n",
    "        \n",
    "        X_train,X_test,y_train,y_test = train_test_split(X,y,test_size =0.2)\n",
    "        X_train,scaler = self.scale_data(X_train)\n",
    "        X_train = scaler.transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        \n",
    "        print(\"Calinng SFS\")\n",
    "     \n",
    "        print(\"**Training of Student_performance starts**\")\n",
    "        \n",
    "        print(\"__Training SVR__\")\n",
    "        \n",
    "        #svr_reg = self.train_reg(svm.SVR(),self.SVR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training DTR__\")\n",
    "        dt_reg = self.train_reg(sklearn.tree.DecisionTreeRegressor(),self.DTR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Random Forest Reg__\")\n",
    "        rf_reg = self.train_reg(sklearn.ensemble.RandomForestRegressor(),self.RF_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training AdaBoost Reg__\")\n",
    "        adb_reg = self.train_reg(sklearn.ensemble.AdaBoostRegressor(),self.ADB_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Gaussian proc Reg__\")\n",
    "        gp_reg = self.train_reg(sklearn.gaussian_process.GaussianProcessRegressor(),self.GP_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "        print(\"__Training Linear Reg__\")\n",
    "        lin_reg = self.train_reg(sklearn.linear_model.LinearRegression(),self.LR_reg(X_train,y_train),X_train,y_train)\n",
    "        \n",
    "      #  print(\"__Training Neural Net Reg__\")\n",
    "       # nn_reg = self.train_reg(sklearn.neural_network.MLPRegressor(),self.MLP_reg(X_train.astype(float),y_train),X_train.astype(float),y_train)\n",
    "        \n",
    "        print(\"**Test Data Prediction Begins*\")\n",
    "        \n",
    "        \n",
    "        #print(\"Testing SVM Regressionr\")\n",
    "        #print(svr_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Decision Trees\")\n",
    "        print(dt_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Random Forests\")\n",
    "        print(rf_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Adaboost\")\n",
    "        print(adb_reg.score(X_test,y_test))\n",
    "        \n",
    "        \n",
    "        print(\"Testing Gaussian Process reg\")\n",
    "        print(gp_reg.score(X_test,y_test))\n",
    "        \n",
    "        print(\"Testing Linear Regression\")\n",
    "        print(lin_reg.score(X_test,y_test))\n",
    "        \n",
    "        #print(\"Testing Neural Network/MLP\")\n",
    "       # print(nn_reg.score(X_test,y_test))\n",
    "        \n",
    "        \n",
    "        return\n",
    "    \n",
    "obj = wine()\n",
    "obj.start()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "4r.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}