random_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import warnings\n",
    "from tqdm import tqdm\n",
    "from evaluator import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "VARIABLES = ['ActivityCounts', 'Barometer', 'BloodPerfusion',\n",
    "             'BloodPulseWave', 'EnergyExpenditure', 'GalvanicSkinResponse', 'HR',\n",
    "             'HRV', 'RESP', 'Steps', 'SkinTemperature', 'ActivityClass']"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Import data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# file path to data folder\n",
    "path = './Output'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Metadata (subjectID, etc.)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "with open(path + '/metadata_stat.txt') as f:\n",
    "    metadata = f.read()\n",
    "\n",
    "metadata = json.loads(metadata.replace('\\'', '\\\"').replace('False', 'false').replace('True', 'true')) # doesn't accept other chars"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "subjects = [meta['subjectID'] for meta in metadata]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Random Forest"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "class RandomForest:\n",
    "\n",
    "    def __init__(self, path, variable):\n",
    "        self.SEED = 42\n",
    "        self.model = None\n",
    "        self.path = path\n",
    "        assert variable in (0, 1)\n",
    "        self.variable = variable\n",
    "        self.normalizer = StandardScaler()\n",
    "\n",
    "        # CV ranges\n",
    "        self.folds = 5\n",
    "        self.n_trees = [3, 10, 50, 100, 300, 1000]\n",
    "        self.max_features = ['auto', 'sqrt', 'log2']\n",
    "        self.max_depths = [10, 30, 50, 100]\n",
    "        self.criterions = ['gini', 'entropy']\n",
    "        self.min_samples_splits = [2, 5, 10]\n",
    "\n",
    "    def load_data(self, indices):\n",
    "        # load shape\n",
    "        N = len(indices)\n",
    "        N_FEATURES = np.load(self.path + '/feature_vector_stat0.npy').shape[0]\n",
    "\n",
    "        # init\n",
    "        X = np.empty((N, N_FEATURES))\n",
    "        y = np.empty(N)\n",
    "\n",
    "        # load individual datapoints\n",
    "        for i, index in enumerate(indices):\n",
    "            X[i, ] = np.load(path + f'/feature_vector_stat{index}.npy', allow_pickle=True)\n",
    "            y[i, ] = np.load(path + f'/labels_stat{index}.npy', allow_pickle=True)[self.variable]\n",
    "\n",
    "        return X, y\n",
    "\n",
    "    def fit(self, train_indices):\n",
    "        # load data\n",
    "        X_train, y_train = self.load_data(train_indices)\n",
    "\n",
    "        # normalize training set\n",
    "        self.normalizer.fit(X_train) # fit accord. to training set\n",
    "        X_train = self.normalizer.transform(X_train, copy=True)\n",
    "\n",
    "        # inner CV (hyperparameter tuning)\n",
    "        inner_cv = StratifiedKFold(n_splits=self.folds, shuffle=True, random_state=self.SEED)\n",
    "        combinations = {}\n",
    "        for n_tree in tqdm(self.n_trees):\n",
    "            for max_feature in self.max_features:\n",
    "                for max_depth in self.max_depths:\n",
    "                    for criterion in self.criterions:\n",
    "                        for min_sample_split in self.min_samples_splits:\n",
    "                            # model\n",
    "                            rf = RandomForestClassifier(n_estimators=n_tree,\n",
    "                                                        criterion=criterion,\n",
    "                                                        max_depth=max_depth,\n",
    "                                                        min_samples_split=min_sample_split,\n",
    "                                                        max_features=max_feature)\n",
    "\n",
    "                            # CV\n",
    "                            scores = cross_val_score(rf, X_train, y_train, cv=inner_cv, scoring='f1_weighted')\n",
    "\n",
    "                            # store score\n",
    "                            combination = (n_tree, max_feature, max_depth, criterion, min_sample_split)\n",
    "                            combinations[combination] = np.mean(scores)\n",
    "\n",
    "        # best hyperparams\n",
    "        best_combination, best_score = sorted(list(combinations.items()), key=lambda item: item[1])[-1]\n",
    "\n",
    "        # use model with best hyperparams\n",
    "        self.model = RandomForestClassifier(n_estimators=best_combination[0],\n",
    "                                            criterion=best_combination[3],\n",
    "                                            max_depth=best_combination[2],\n",
    "                                            min_samples_split=best_combination[4],\n",
    "                                            max_features=best_combination[1])\n",
    "\n",
    "        self.model.fit(X_train, y_train)\n",
    "\n",
    "    def predict(self, test_indices):\n",
    "        # load data\n",
    "        X_test, _ = self.load_data(test_indices)\n",
    "\n",
    "        # normalize test set\n",
    "        X_test = self.normalizer.transform(X_test, copy=True)\n",
    "\n",
    "        return self.model.predict(X_test)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# CV"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting stratified group 5-fold for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.71s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.20s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:45<00:57, 19.16s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:56<01:18, 39.47s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:30<01:42, 102.57s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:18<00:00, 173.03s/it]\u001B[A\n",
      " Fold 1 F1: 0.36651583710407243:  20%|██        | 1/5 [17:18<1:09:14, 1038.57s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.12s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.64s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:53, 17.71s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:46<01:11, 35.85s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:03<01:34, 94.04s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:48<00:00, 158.04s/it]\u001B[A\n",
      " Fold 2 F1: 0.5978589322153203:  40%|████      | 2/5 [33:08<49:19, 986.66s/it]    \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.08s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:23,  5.77s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:44<00:56, 18.74s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:49<01:14, 37.03s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:09<01:35, 95.56s/it]\u001B[A\n",
      "100%|██████████| 6/6 [16:22<00:00, 163.82s/it]\u001B[A\n",
      " Fold 3 F1: 0.4590626764539808:  60%|██████    | 3/5 [49:32<32:50, 985.07s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.85s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.30s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:40<00:50, 16.85s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:41<01:08, 34.26s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:38<01:25, 85.72s/it]\u001B[A\n",
      "100%|██████████| 6/6 [14:30<00:00, 145.10s/it]\u001B[A\n",
      " Fold 4 F1: 0.7586617826617826:  80%|████████  | 4/5 [1:04:02<15:39, 939.98s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.63s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.30s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:41<00:52, 17.53s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:45<01:11, 35.59s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:15<01:38, 98.55s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:46<00:00, 177.74s/it]\u001B[A\n",
      " Fold 5 F1: 0.9157595249876298: 100%|██████████| 5/5 [1:21:49<00:00, 981.93s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.65 +- 0.152 \n",
      "\n",
      " balanced_accuracy: 0.596 +- 0.174 \n",
      "\n",
      " f1: 0.62 +- 0.199 \n",
      "\n",
      " recall: 0.65 +- 0.152 \n",
      "\n",
      " precision: 0.629 +- 0.237 \n",
      "\n",
      "Starting stratified 5-fold for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.34s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:23,  5.84s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:41<00:51, 17.08s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:41<01:08, 34.17s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:20<01:40, 100.76s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:52<00:00, 178.68s/it]\u001B[A\n",
      " Fold 1 F1: 0.6784601691143748:  20%|██        | 1/5 [17:52<1:11:29, 1072.49s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.34s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:25,  6.37s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:50<01:02, 20.98s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:05<01:24, 42.37s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:49<01:48, 108.03s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:48<00:00, 178.01s/it]\u001B[A\n",
      " Fold 2 F1: 0.7532051282051282:  40%|████      | 2/5 [35:41<53:30, 1070.28s/it]  \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.13s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.08s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:48<01:00, 20.22s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:04<01:24, 42.12s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:45<01:46, 106.75s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:59<00:00, 179.99s/it]\u001B[A\n",
      " Fold 3 F1: 0.7096237096237096:  60%|██████    | 3/5 [53:42<35:51, 1075.52s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.39s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.24s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:49<01:01, 20.47s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:02<01:23, 41.51s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:51<01:49, 109.14s/it]\u001B[A\n",
      "100%|██████████| 6/6 [18:40<00:00, 186.67s/it]\u001B[A\n",
      " Fold 4 F1: 0.8027210884353743:  80%|████████  | 4/5 [1:12:23<18:13, 1093.22s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.16s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:25,  6.38s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:50<01:02, 20.97s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:05<01:24, 42.46s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:50<01:48, 108.13s/it]\u001B[A\n",
      "100%|██████████| 6/6 [18:28<00:00, 184.81s/it]\u001B[A\n",
      " Fold 5 F1: 0.719256052589386: 100%|██████████| 5/5 [1:30:52<00:00, 1090.50s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.757 +- 0.026 \n",
      "\n",
      " balanced_accuracy: 0.616 +- 0.085 \n",
      "\n",
      " f1: 0.733 +- 0.042 \n",
      "\n",
      " recall: 0.757 +- 0.026 \n",
      "\n",
      " precision: 0.733 +- 0.053 \n",
      "\n",
      "Starting leave-one-subject-out for physical fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/21 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:18,  3.66s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:13<00:28,  7.05s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:54<01:07, 22.63s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:16<01:32, 46.09s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:59<01:49, 109.91s/it]\u001B[A\n",
      "100%|██████████| 6/6 [18:50<00:00, 188.47s/it]\u001B[A\n",
      " Fold 1 F1: 0.40476190476190477:   5%|▍         | 1/21 [18:51<6:17:00, 1131.03s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:17,  3.52s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:12<00:27,  6.85s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:53<01:06, 22.21s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:12<01:29, 44.86s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [06:08<01:53, 113.56s/it]\u001B[A\n",
      "100%|██████████| 6/6 [18:14<00:00, 182.41s/it]\u001B[A\n",
      " Fold 2 F1: 0.16666666666666666:  10%|▉         | 2/21 [37:05<5:51:23, 1109.68s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.10s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.14s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:48<01:00, 20.30s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:00<01:21, 40.84s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:34<01:42, 102.98s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:27<00:00, 174.65s/it]\u001B[A\n",
      " Fold 3 F1: 0.4:  14%|█▍        | 3/21 [54:33<5:24:28, 1081.57s/it]                \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  3.00s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.03s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<01:00, 20.03s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:02<01:23, 41.60s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:58<01:51, 111.55s/it]\u001B[A\n",
      "100%|██████████| 6/6 [19:29<00:00, 194.91s/it]\u001B[A\n",
      " Fold 4 F1: 1.0:  19%|█▉        | 4/21 [1:14:03<5:16:18, 1116.36s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.10s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.18s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:53<01:08, 22.77s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:18<01:34, 47.18s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [06:22<01:58, 118.13s/it]\u001B[A\n",
      "100%|██████████| 6/6 [19:11<00:00, 191.84s/it]\u001B[A\n",
      " Fold 5 F1: 0.5333333333333333:  24%|██▍       | 5/21 [1:33:14<5:01:03, 1128.99s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.08s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.08s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:48<01:00, 20.08s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:00<01:21, 40.91s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:37<01:44, 104.38s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:25<00:00, 174.27s/it]\u001B[A\n",
      " Fold 6 F1: 0.30000000000000004:  29%|██▊       | 6/21 [1:50:40<4:35:10, 1100.73s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.23s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.15s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:48<01:00, 20.27s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:01<01:22, 41.10s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:37<01:44, 104.30s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:31<00:00, 175.22s/it]\u001B[A\n",
      " Fold 7 F1: 0.0:  33%|███▎      | 7/21 [2:08:12<4:13:05, 1084.64s/it]                \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.05s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.05s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<00:59, 19.89s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:59<01:20, 40.29s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:30<01:41, 101.97s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:09<00:00, 171.56s/it]\u001B[A\n",
      " Fold 8 F1: 0.25:  38%|███▊      | 8/21 [2:25:21<3:51:12, 1067.13s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.04s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.04s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<00:59, 19.90s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:59<01:20, 40.34s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:30<01:41, 101.81s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:16<00:00, 172.74s/it]\u001B[A\n",
      " Fold 9 F1: 0.7272727272727273:  43%|████▎     | 9/21 [2:42:38<3:31:31, 1057.61s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.95s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.08s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<00:59, 19.94s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:00<01:21, 40.69s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:32<01:42, 102.48s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:24<00:00, 174.01s/it]\u001B[A\n",
      " Fold 10 F1: 0.5:  48%|████▊     | 10/21 [3:00:02<3:13:08, 1053.50s/it]              \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.26s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.10s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:48<01:00, 20.18s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:57<01:19, 39.50s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:08<01:34, 94.08s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:41<00:00, 156.86s/it]\u001B[A\n",
      " Fold 11 F1: 0.7111111111111111:  52%|█████▏    | 11/21 [3:15:44<2:49:51, 1019.18s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.62s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.29s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:52, 17.65s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:45<01:11, 35.85s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:55<01:31, 91.34s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:25<00:00, 154.30s/it]\u001B[A\n",
      " Fold 12 F1: 0.876984126984127:  57%|█████▋    | 12/21 [3:31:10<2:28:37, 990.84s/it]  \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.73s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.35s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:53, 17.67s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:45<01:11, 35.84s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:55<01:31, 91.22s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:26<00:00, 154.37s/it]\u001B[A\n",
      " Fold 13 F1: 1.0:  62%|██████▏   | 13/21 [3:46:36<2:09:30, 971.33s/it]              \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.73s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.30s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:53, 17.72s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:46<01:12, 36.06s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:57<01:31, 91.86s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:31<00:00, 155.31s/it]\u001B[A\n",
      " Fold 14 F1: 1.0:  67%|██████▋   | 14/21 [4:02:08<1:51:56, 959.47s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.99s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.67s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:44<00:55, 18.41s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:49<01:13, 36.94s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:20<01:39, 99.77s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:48<00:00, 178.08s/it]\u001B[A\n",
      " Fold 15 F1: 0.4047619047619047:  71%|███████▏  | 15/21 [4:19:57<1:39:14, 992.40s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.30s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.23s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<00:59, 19.94s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:58<01:19, 39.88s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:37<01:44, 104.72s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:42<00:00, 177.08s/it]\u001B[A\n",
      " Fold 16 F1: 0.8571428571428571:  76%|███████▌  | 16/21 [4:37:40<1:24:27, 1013.56s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.69s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:20,  5.06s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:53, 17.68s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:47<01:12, 36.35s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:00<01:32, 92.92s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:34<00:00, 155.78s/it]\u001B[A\n",
      " Fold 17 F1: 0.4840634920634921:  81%|████████  | 17/21 [4:53:15<1:05:59, 989.92s/it] \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.80s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.58s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:54, 18.20s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:48<01:13, 36.58s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:00<01:32, 92.83s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:38<00:00, 156.46s/it]\u001B[A\n",
      " Fold 18 F1: 0.8796026210103572:  86%|████████▌ | 18/21 [5:08:54<48:43, 974.62s/it]  \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.11s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.00s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:46<00:57, 19.23s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:54<01:17, 38.74s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:25<01:40, 100.72s/it]\u001B[A\n",
      "100%|██████████| 6/6 [18:20<00:00, 183.38s/it]\u001B[A\n",
      " Fold 19 F1: 0.36651583710407243:  90%|█████████ | 19/21 [5:27:14<33:44, 1012.43s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:17,  3.42s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:12<00:26,  6.70s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:54<01:08, 22.73s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:14<01:30, 45.32s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [06:12<01:54, 114.96s/it]\u001B[A\n",
      "100%|██████████| 6/6 [19:07<00:00, 191.29s/it]\u001B[A\n",
      " Fold 20 F1: 0.7750404379743686:  95%|█████████▌| 20/21 [5:46:22<17:33, 1053.13s/it] \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.29s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.24s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:49<01:01, 20.59s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [02:03<01:23, 41.73s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:44<01:46, 106.38s/it]\u001B[A\n",
      "100%|██████████| 6/6 [17:38<00:00, 176.49s/it]\u001B[A\n",
      " Fold 21 F1: 0.41666666666666663: 100%|██████████| 21/21 [6:04:01<00:00, 1040.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.575 +- 0.274 \n",
      "\n",
      " balanced_accuracy: 0.562 +- 0.282 \n",
      "\n",
      " f1: 0.574 +- 0.288 \n",
      "\n",
      " recall: 0.575 +- 0.274 \n",
      "\n",
      " precision: 0.642 +- 0.324 \n",
      "\n",
      "Starting stratified group 5-fold for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:16,  3.26s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:11<00:24,  6.08s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:47<00:59, 19.67s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:57<01:19, 39.79s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:28<01:41, 101.52s/it]\u001B[A\n",
      "100%|██████████| 6/6 [16:58<00:00, 169.74s/it]\u001B[A\n",
      " Fold 1 F1: 0.3388235294117647:  20%|██        | 1/5 [16:58<1:07:54, 1018.69s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.65s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.29s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:40<00:51, 17.07s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:42<01:09, 34.58s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:43<01:27, 87.56s/it]\u001B[A\n",
      "100%|██████████| 6/6 [14:43<00:00, 147.19s/it]\u001B[A\n",
      " Fold 2 F1: 0.4557755131182846:  40%|████      | 2/5 [31:42<46:57, 939.10s/it]   \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.06s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:23,  5.81s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:53, 17.95s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:50<01:14, 37.42s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:14<01:37, 97.63s/it]\u001B[A\n",
      "100%|██████████| 6/6 [16:29<00:00, 164.86s/it]\u001B[A\n",
      " Fold 3 F1: 0.6848759906177568:  60%|██████    | 3/5 [48:11<32:04, 962.09s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.66s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:20,  5.24s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:40<00:50, 16.95s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:43<01:10, 35.19s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:48<01:29, 89.06s/it]\u001B[A\n",
      "100%|██████████| 6/6 [14:54<00:00, 149.04s/it]\u001B[A\n",
      " Fold 4 F1: 0.6453290712321716:  80%|████████  | 4/5 [1:03:05<15:35, 935.38s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.02s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:23,  5.87s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:46<00:58, 19.37s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:56<01:18, 39.38s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:18<01:38, 98.11s/it]\u001B[A\n",
      "100%|██████████| 6/6 [16:35<00:00, 165.95s/it]\u001B[A\n",
      " Fold 5 F1: 0.3482544442064682: 100%|██████████| 5/5 [1:19:41<00:00, 956.38s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.537 +- 0.083 \n",
      "\n",
      " balanced_accuracy: 0.471 +- 0.033 \n",
      "\n",
      " f1: 0.495 +- 0.146 \n",
      "\n",
      " recall: 0.537 +- 0.083 \n",
      "\n",
      " precision: 0.564 +- 0.19 \n",
      "\n",
      "Starting stratified 5-fold for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.91s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.60s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:55, 18.33s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:50<01:14, 37.23s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:03<01:33, 93.64s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:51<00:00, 158.66s/it]\u001B[A\n",
      " Fold 1 F1: 0.5872926093514328:  20%|██        | 1/5 [15:52<1:03:28, 952.22s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:03<00:15,  3.09s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.62s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:44<00:55, 18.48s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:50<01:15, 37.53s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:07<01:34, 94.93s/it]\u001B[A\n",
      "100%|██████████| 6/6 [16:01<00:00, 160.17s/it]\u001B[A\n",
      " Fold 2 F1: 0.7159888357256778:  40%|████      | 2/5 [31:53<47:52, 957.55s/it]  \n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.99s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.62s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:54, 18.22s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:49<01:13, 36.88s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:02<01:33, 93.45s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:41<00:00, 156.89s/it]\u001B[A\n",
      " Fold 3 F1: 0.7169700072925879:  60%|██████    | 3/5 [47:35<31:40, 950.27s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.89s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:21,  5.48s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:53, 17.98s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:48<01:13, 36.66s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:01<01:32, 92.97s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:38<00:00, 156.34s/it]\u001B[A\n",
      " Fold 4 F1: 0.5729916188320264:  80%|████████  | 4/5 [1:03:13<15:45, 945.53s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:14,  2.93s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:10<00:22,  5.62s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:43<00:54, 18.31s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:49<01:14, 37.09s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [05:05<01:34, 94.17s/it]\u001B[A\n",
      "100%|██████████| 6/6 [14:38<00:00, 146.41s/it]\u001B[A\n",
      " Fold 5 F1: 0.6172839506172839: 100%|██████████| 5/5 [1:17:52<00:00, 934.41s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance model:\n",
      " accuracy: 0.678 +- 0.049 \n",
      "\n",
      " balanced_accuracy: 0.558 +- 0.071 \n",
      "\n",
      " f1: 0.642 +- 0.062 \n",
      "\n",
      " recall: 0.678 +- 0.049 \n",
      "\n",
      " precision: 0.635 +- 0.069 \n",
      "\n",
      "Starting leave-one-subject-out for mental fatigue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/21 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.69s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.29s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:52, 17.61s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:45<01:11, 35.75s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:54<01:30, 90.88s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:21<00:00, 153.62s/it]\u001B[A\n",
      " Fold 1 F1: 0.6309523809523809:   5%|▍         | 1/21 [15:21<5:07:18, 921.94s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.68s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.27s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:41<00:52, 17.53s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:45<01:11, 35.64s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [04:53<01:30, 90.60s/it]\u001B[A\n",
      "100%|██████████| 6/6 [15:19<00:00, 153.29s/it]\u001B[A\n",
      " Fold 2 F1: 0.3333333333333333:  10%|▉         | 2/21 [30:41<4:51:34, 920.78s/it]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001B[A\n",
      " 17%|█▋        | 1/6 [00:02<00:13,  2.63s/it]\u001B[A\n",
      " 33%|███▎      | 2/6 [00:09<00:21,  5.31s/it]\u001B[A\n",
      " 50%|█████     | 3/6 [00:42<00:53, 17.77s/it]\u001B[A\n",
      " 67%|██████▋   | 4/6 [01:46<01:12, 36.05s/it]\u001B[A\n",
      " 83%|████████▎ | 5/6 [12:49<02:33, 153.90s/it][A\n",
      " Fold 2 F1: 0.3333333333333333:  10%|▉         | 2/21 [43:31<6:53:30, 1305.80s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn [7], line 26\u001B[0m\n\u001B[0;32m     11\u001B[0m scores_strat_group_k_fold[variable] \u001B[38;5;241m=\u001B[39m stratified_group_k_fold(path\u001B[38;5;241m=\u001B[39mpath,\n\u001B[0;32m     12\u001B[0m                                                     groups\u001B[38;5;241m=\u001B[39msubjects,\n\u001B[0;32m     13\u001B[0m                                                     model\u001B[38;5;241m=\u001B[39mmodel,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     16\u001B[0m                                                     verbose\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[0;32m     17\u001B[0m                                                     variable\u001B[38;5;241m=\u001B[39mvariable)\n\u001B[0;32m     19\u001B[0m scores_strat_k_fold[variable] \u001B[38;5;241m=\u001B[39m stratified_k_fold(path\u001B[38;5;241m=\u001B[39mpath,\n\u001B[0;32m     20\u001B[0m                                         model\u001B[38;5;241m=\u001B[39mmodel,\n\u001B[0;32m     21\u001B[0m                                         folds\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m5\u001B[39m,\n\u001B[0;32m     22\u001B[0m                                         images\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m     23\u001B[0m                                         verbose\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[0;32m     24\u001B[0m                                         variable\u001B[38;5;241m=\u001B[39mvariable)\n\u001B[1;32m---> 26\u001B[0m scores_loso[variable] \u001B[38;5;241m=\u001B[39m \u001B[43mleave_one_subject_out\u001B[49m\u001B[43m(\u001B[49m\u001B[43mpath\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mpath\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m     27\u001B[0m \u001B[43m                                    \u001B[49m\u001B[43mgroups\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msubjects\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m     28\u001B[0m \u001B[43m                                    \u001B[49m\u001B[43mmodel\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m     29\u001B[0m \u001B[43m                                    \u001B[49m\u001B[43mimages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m     30\u001B[0m \u001B[43m                                    \u001B[49m\u001B[43mverbose\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m     31\u001B[0m \u001B[43m                                    \u001B[49m\u001B[43mvariable\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mvariable\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32m~\\OneDrive\\Dokumente\\ETH\\MSc 3rd semester\\Semester project\\evaluator.py:282\u001B[0m, in \u001B[0;36mleave_one_subject_out\u001B[1;34m(path, groups, model, images, verbose, variable)\u001B[0m\n\u001B[0;32m    280\u001B[0m     model\u001B[38;5;241m.\u001B[39mreset()\n\u001B[0;32m    281\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mAttributeError\u001B[39;00m:\n\u001B[1;32m--> 282\u001B[0m     \u001B[38;5;28;01mpass\u001B[39;00m  \u001B[38;5;66;03m# model has no trainable parameters\u001B[39;00m\n\u001B[0;32m    283\u001B[0m model\u001B[38;5;241m.\u001B[39mfit(train_indices)\n\u001B[0;32m    285\u001B[0m \u001B[38;5;66;03m# predict\u001B[39;00m\n",
      "Cell \u001B[1;32mIn [6], line 59\u001B[0m, in \u001B[0;36mRandomForest.fit\u001B[1;34m(self, train_indices)\u001B[0m\n\u001B[0;32m     52\u001B[0m rf \u001B[38;5;241m=\u001B[39m RandomForestClassifier(n_estimators\u001B[38;5;241m=\u001B[39mn_tree,\n\u001B[0;32m     53\u001B[0m                             criterion\u001B[38;5;241m=\u001B[39mcriterion,\n\u001B[0;32m     54\u001B[0m                             max_depth\u001B[38;5;241m=\u001B[39mmax_depth,\n\u001B[0;32m     55\u001B[0m                             min_samples_split\u001B[38;5;241m=\u001B[39mmin_sample_split,\n\u001B[0;32m     56\u001B[0m                             max_features\u001B[38;5;241m=\u001B[39mmax_feature)\n\u001B[0;32m     58\u001B[0m \u001B[38;5;66;03m# CV\u001B[39;00m\n\u001B[1;32m---> 59\u001B[0m scores \u001B[38;5;241m=\u001B[39m \u001B[43mcross_val_score\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mX_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcv\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minner_cv\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mscoring\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mf1_weighted\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[0;32m     61\u001B[0m \u001B[38;5;66;03m# store score\u001B[39;00m\n\u001B[0;32m     62\u001B[0m combination \u001B[38;5;241m=\u001B[39m (n_tree, max_feature, max_depth, criterion, min_sample_split)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:515\u001B[0m, in \u001B[0;36mcross_val_score\u001B[1;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, error_score)\u001B[0m\n\u001B[0;32m    512\u001B[0m \u001B[38;5;66;03m# To ensure multimetric format is not supported\u001B[39;00m\n\u001B[0;32m    513\u001B[0m scorer \u001B[38;5;241m=\u001B[39m check_scoring(estimator, scoring\u001B[38;5;241m=\u001B[39mscoring)\n\u001B[1;32m--> 515\u001B[0m cv_results \u001B[38;5;241m=\u001B[39m \u001B[43mcross_validate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    516\u001B[0m \u001B[43m    \u001B[49m\u001B[43mestimator\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mestimator\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    517\u001B[0m \u001B[43m    \u001B[49m\u001B[43mX\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    518\u001B[0m \u001B[43m    \u001B[49m\u001B[43my\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    519\u001B[0m \u001B[43m    \u001B[49m\u001B[43mgroups\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mgroups\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    520\u001B[0m \u001B[43m    \u001B[49m\u001B[43mscoring\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m{\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mscore\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[43mscorer\u001B[49m\u001B[43m}\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    521\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcv\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcv\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    522\u001B[0m \u001B[43m    \u001B[49m\u001B[43mn_jobs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_jobs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    523\u001B[0m \u001B[43m    \u001B[49m\u001B[43mverbose\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mverbose\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    524\u001B[0m \u001B[43m    \u001B[49m\u001B[43mfit_params\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfit_params\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    525\u001B[0m \u001B[43m    \u001B[49m\u001B[43mpre_dispatch\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mpre_dispatch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    526\u001B[0m \u001B[43m    \u001B[49m\u001B[43merror_score\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43merror_score\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    527\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    528\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m cv_results[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtest_score\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:266\u001B[0m, in \u001B[0;36mcross_validate\u001B[1;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001B[0m\n\u001B[0;32m    263\u001B[0m \u001B[38;5;66;03m# We clone the estimator to make sure that all the folds are\u001B[39;00m\n\u001B[0;32m    264\u001B[0m \u001B[38;5;66;03m# independent, and that it is pickle-able.\u001B[39;00m\n\u001B[0;32m    265\u001B[0m parallel \u001B[38;5;241m=\u001B[39m Parallel(n_jobs\u001B[38;5;241m=\u001B[39mn_jobs, verbose\u001B[38;5;241m=\u001B[39mverbose, pre_dispatch\u001B[38;5;241m=\u001B[39mpre_dispatch)\n\u001B[1;32m--> 266\u001B[0m results \u001B[38;5;241m=\u001B[39m \u001B[43mparallel\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    267\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdelayed\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_fit_and_score\u001B[49m\u001B[43m)\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    268\u001B[0m \u001B[43m        \u001B[49m\u001B[43mclone\u001B[49m\u001B[43m(\u001B[49m\u001B[43mestimator\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    269\u001B[0m \u001B[43m        \u001B[49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    270\u001B[0m \u001B[43m        \u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    271\u001B[0m \u001B[43m        \u001B[49m\u001B[43mscorers\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    272\u001B[0m \u001B[43m        \u001B[49m\u001B[43mtrain\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    273\u001B[0m \u001B[43m        \u001B[49m\u001B[43mtest\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    274\u001B[0m \u001B[43m        \u001B[49m\u001B[43mverbose\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    275\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m    276\u001B[0m \u001B[43m        \u001B[49m\u001B[43mfit_params\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    277\u001B[0m \u001B[43m        \u001B[49m\u001B[43mreturn_train_score\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreturn_train_score\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    278\u001B[0m \u001B[43m        \u001B[49m\u001B[43mreturn_times\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[0;32m    279\u001B[0m \u001B[43m        \u001B[49m\u001B[43mreturn_estimator\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreturn_estimator\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    280\u001B[0m \u001B[43m        \u001B[49m\u001B[43merror_score\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43merror_score\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    281\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    282\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mtrain\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtest\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mcv\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msplit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgroups\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    283\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    285\u001B[0m _warn_or_raise_about_fit_failures(results, error_score)\n\u001B[0;32m    287\u001B[0m \u001B[38;5;66;03m# For callabe scoring, the return type is only know after calling. If the\u001B[39;00m\n\u001B[0;32m    288\u001B[0m \u001B[38;5;66;03m# return type is a dictionary, the error scores can now be inserted with\u001B[39;00m\n\u001B[0;32m    289\u001B[0m \u001B[38;5;66;03m# the correct key.\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:1051\u001B[0m, in \u001B[0;36mParallel.__call__\u001B[1;34m(self, iterable)\u001B[0m\n\u001B[0;32m   1048\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdispatch_one_batch(iterator):\n\u001B[0;32m   1049\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterating \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_original_iterator \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m-> 1051\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdispatch_one_batch\u001B[49m\u001B[43m(\u001B[49m\u001B[43miterator\u001B[49m\u001B[43m)\u001B[49m:\n\u001B[0;32m   1052\u001B[0m     \u001B[38;5;28;01mpass\u001B[39;00m\n\u001B[0;32m   1054\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m pre_dispatch \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mall\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m n_jobs \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m   1055\u001B[0m     \u001B[38;5;66;03m# The iterable was consumed all at once by the above for loop.\u001B[39;00m\n\u001B[0;32m   1056\u001B[0m     \u001B[38;5;66;03m# No need to wait for async callbacks to trigger to\u001B[39;00m\n\u001B[0;32m   1057\u001B[0m     \u001B[38;5;66;03m# consumption.\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:864\u001B[0m, in \u001B[0;36mParallel.dispatch_one_batch\u001B[1;34m(self, iterator)\u001B[0m\n\u001B[0;32m    862\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[0;32m    863\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 864\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_dispatch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtasks\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    865\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:782\u001B[0m, in \u001B[0;36mParallel._dispatch\u001B[1;34m(self, batch)\u001B[0m\n\u001B[0;32m    780\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_lock:\n\u001B[0;32m    781\u001B[0m     job_idx \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_jobs)\n\u001B[1;32m--> 782\u001B[0m     job \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_backend\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mapply_async\u001B[49m\u001B[43m(\u001B[49m\u001B[43mbatch\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcallback\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    783\u001B[0m     \u001B[38;5;66;03m# A job can complete so quickly than its callback is\u001B[39;00m\n\u001B[0;32m    784\u001B[0m     \u001B[38;5;66;03m# called before we get here, causing self._jobs to\u001B[39;00m\n\u001B[0;32m    785\u001B[0m     \u001B[38;5;66;03m# grow. To ensure correct results ordering, .insert is\u001B[39;00m\n\u001B[0;32m    786\u001B[0m     \u001B[38;5;66;03m# used (rather than .append) in the following line\u001B[39;00m\n\u001B[0;32m    787\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_jobs\u001B[38;5;241m.\u001B[39minsert(job_idx, job)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\_parallel_backends.py:208\u001B[0m, in \u001B[0;36mSequentialBackend.apply_async\u001B[1;34m(self, func, callback)\u001B[0m\n\u001B[0;32m    206\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mapply_async\u001B[39m(\u001B[38;5;28mself\u001B[39m, func, callback\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[0;32m    207\u001B[0m     \u001B[38;5;124;03m\"\"\"Schedule a func to be run\"\"\"\u001B[39;00m\n\u001B[1;32m--> 208\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43mImmediateResult\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    209\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m callback:\n\u001B[0;32m    210\u001B[0m         callback(result)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\_parallel_backends.py:572\u001B[0m, in \u001B[0;36mImmediateResult.__init__\u001B[1;34m(self, batch)\u001B[0m\n\u001B[0;32m    569\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__init__\u001B[39m(\u001B[38;5;28mself\u001B[39m, batch):\n\u001B[0;32m    570\u001B[0m     \u001B[38;5;66;03m# Don't delay the application, to avoid keeping the input\u001B[39;00m\n\u001B[0;32m    571\u001B[0m     \u001B[38;5;66;03m# arguments in memory\u001B[39;00m\n\u001B[1;32m--> 572\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mresults \u001B[38;5;241m=\u001B[39m \u001B[43mbatch\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:263\u001B[0m, in \u001B[0;36mBatchedCalls.__call__\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    259\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m    260\u001B[0m     \u001B[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001B[39;00m\n\u001B[0;32m    261\u001B[0m     \u001B[38;5;66;03m# change the default number of processes to -1\u001B[39;00m\n\u001B[0;32m    262\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m parallel_backend(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend, n_jobs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_n_jobs):\n\u001B[1;32m--> 263\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m [func(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    264\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m func, args, kwargs \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mitems]\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:263\u001B[0m, in \u001B[0;36m<listcomp>\u001B[1;34m(.0)\u001B[0m\n\u001B[0;32m    259\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m    260\u001B[0m     \u001B[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001B[39;00m\n\u001B[0;32m    261\u001B[0m     \u001B[38;5;66;03m# change the default number of processes to -1\u001B[39;00m\n\u001B[0;32m    262\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m parallel_backend(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend, n_jobs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_n_jobs):\n\u001B[1;32m--> 263\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m [func(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    264\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m func, args, kwargs \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mitems]\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\utils\\fixes.py:117\u001B[0m, in \u001B[0;36m_FuncWrapper.__call__\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m    115\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m    116\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m config_context(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mconfig):\n\u001B[1;32m--> 117\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfunction(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:686\u001B[0m, in \u001B[0;36m_fit_and_score\u001B[1;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, split_progress, candidate_progress, error_score)\u001B[0m\n\u001B[0;32m    684\u001B[0m         estimator\u001B[38;5;241m.\u001B[39mfit(X_train, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mfit_params)\n\u001B[0;32m    685\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 686\u001B[0m         estimator\u001B[38;5;241m.\u001B[39mfit(X_train, y_train, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mfit_params)\n\u001B[0;32m    688\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m:\n\u001B[0;32m    689\u001B[0m     \u001B[38;5;66;03m# Note fit time as time until error\u001B[39;00m\n\u001B[0;32m    690\u001B[0m     fit_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m-\u001B[39m start_time\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\ensemble\\_forest.py:476\u001B[0m, in \u001B[0;36mBaseForest.fit\u001B[1;34m(self, X, y, sample_weight)\u001B[0m\n\u001B[0;32m    465\u001B[0m trees \u001B[38;5;241m=\u001B[39m [\n\u001B[0;32m    466\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_make_estimator(append\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m, random_state\u001B[38;5;241m=\u001B[39mrandom_state)\n\u001B[0;32m    467\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m i \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(n_more_estimators)\n\u001B[0;32m    468\u001B[0m ]\n\u001B[0;32m    470\u001B[0m \u001B[38;5;66;03m# Parallel loop: we prefer the threading backend as the Cython code\u001B[39;00m\n\u001B[0;32m    471\u001B[0m \u001B[38;5;66;03m# for fitting the trees is internally releasing the Python GIL\u001B[39;00m\n\u001B[0;32m    472\u001B[0m \u001B[38;5;66;03m# making threading more efficient than multiprocessing in\u001B[39;00m\n\u001B[0;32m    473\u001B[0m \u001B[38;5;66;03m# that case. However, for joblib 0.12+ we respect any\u001B[39;00m\n\u001B[0;32m    474\u001B[0m \u001B[38;5;66;03m# parallel_backend contexts set at a higher level,\u001B[39;00m\n\u001B[0;32m    475\u001B[0m \u001B[38;5;66;03m# since correctness does not rely on using threads.\u001B[39;00m\n\u001B[1;32m--> 476\u001B[0m trees \u001B[38;5;241m=\u001B[39m \u001B[43mParallel\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    477\u001B[0m \u001B[43m    \u001B[49m\u001B[43mn_jobs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mn_jobs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    478\u001B[0m \u001B[43m    \u001B[49m\u001B[43mverbose\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mverbose\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    479\u001B[0m \u001B[43m    \u001B[49m\u001B[43mprefer\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mthreads\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[0;32m    480\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    481\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdelayed\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_parallel_build_trees\u001B[49m\u001B[43m)\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    482\u001B[0m \u001B[43m        \u001B[49m\u001B[43mt\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    483\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbootstrap\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    484\u001B[0m \u001B[43m        \u001B[49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    485\u001B[0m \u001B[43m        \u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    486\u001B[0m \u001B[43m        \u001B[49m\u001B[43msample_weight\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    487\u001B[0m \u001B[43m        \u001B[49m\u001B[43mi\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    488\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;28;43mlen\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mtrees\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    489\u001B[0m \u001B[43m        \u001B[49m\u001B[43mverbose\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mverbose\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    490\u001B[0m \u001B[43m        \u001B[49m\u001B[43mclass_weight\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mclass_weight\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    491\u001B[0m \u001B[43m        \u001B[49m\u001B[43mn_samples_bootstrap\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mn_samples_bootstrap\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    492\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    493\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mi\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mt\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;28;43menumerate\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mtrees\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    494\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    496\u001B[0m \u001B[38;5;66;03m# Collect newly grown trees\u001B[39;00m\n\u001B[0;32m    497\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mestimators_\u001B[38;5;241m.\u001B[39mextend(trees)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:1051\u001B[0m, in \u001B[0;36mParallel.__call__\u001B[1;34m(self, iterable)\u001B[0m\n\u001B[0;32m   1048\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdispatch_one_batch(iterator):\n\u001B[0;32m   1049\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterating \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_original_iterator \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m-> 1051\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdispatch_one_batch\u001B[49m\u001B[43m(\u001B[49m\u001B[43miterator\u001B[49m\u001B[43m)\u001B[49m:\n\u001B[0;32m   1052\u001B[0m     \u001B[38;5;28;01mpass\u001B[39;00m\n\u001B[0;32m   1054\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m pre_dispatch \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mall\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m n_jobs \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m   1055\u001B[0m     \u001B[38;5;66;03m# The iterable was consumed all at once by the above for loop.\u001B[39;00m\n\u001B[0;32m   1056\u001B[0m     \u001B[38;5;66;03m# No need to wait for async callbacks to trigger to\u001B[39;00m\n\u001B[0;32m   1057\u001B[0m     \u001B[38;5;66;03m# consumption.\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:864\u001B[0m, in \u001B[0;36mParallel.dispatch_one_batch\u001B[1;34m(self, iterator)\u001B[0m\n\u001B[0;32m    862\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[0;32m    863\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 864\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_dispatch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtasks\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    865\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:782\u001B[0m, in \u001B[0;36mParallel._dispatch\u001B[1;34m(self, batch)\u001B[0m\n\u001B[0;32m    780\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_lock:\n\u001B[0;32m    781\u001B[0m     job_idx \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_jobs)\n\u001B[1;32m--> 782\u001B[0m     job \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_backend\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mapply_async\u001B[49m\u001B[43m(\u001B[49m\u001B[43mbatch\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcallback\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    783\u001B[0m     \u001B[38;5;66;03m# A job can complete so quickly than its callback is\u001B[39;00m\n\u001B[0;32m    784\u001B[0m     \u001B[38;5;66;03m# called before we get here, causing self._jobs to\u001B[39;00m\n\u001B[0;32m    785\u001B[0m     \u001B[38;5;66;03m# grow. To ensure correct results ordering, .insert is\u001B[39;00m\n\u001B[0;32m    786\u001B[0m     \u001B[38;5;66;03m# used (rather than .append) in the following line\u001B[39;00m\n\u001B[0;32m    787\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_jobs\u001B[38;5;241m.\u001B[39minsert(job_idx, job)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\_parallel_backends.py:208\u001B[0m, in \u001B[0;36mSequentialBackend.apply_async\u001B[1;34m(self, func, callback)\u001B[0m\n\u001B[0;32m    206\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mapply_async\u001B[39m(\u001B[38;5;28mself\u001B[39m, func, callback\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[0;32m    207\u001B[0m     \u001B[38;5;124;03m\"\"\"Schedule a func to be run\"\"\"\u001B[39;00m\n\u001B[1;32m--> 208\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43mImmediateResult\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfunc\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    209\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m callback:\n\u001B[0;32m    210\u001B[0m         callback(result)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\_parallel_backends.py:572\u001B[0m, in \u001B[0;36mImmediateResult.__init__\u001B[1;34m(self, batch)\u001B[0m\n\u001B[0;32m    569\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__init__\u001B[39m(\u001B[38;5;28mself\u001B[39m, batch):\n\u001B[0;32m    570\u001B[0m     \u001B[38;5;66;03m# Don't delay the application, to avoid keeping the input\u001B[39;00m\n\u001B[0;32m    571\u001B[0m     \u001B[38;5;66;03m# arguments in memory\u001B[39;00m\n\u001B[1;32m--> 572\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mresults \u001B[38;5;241m=\u001B[39m \u001B[43mbatch\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:263\u001B[0m, in \u001B[0;36mBatchedCalls.__call__\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    259\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m    260\u001B[0m     \u001B[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001B[39;00m\n\u001B[0;32m    261\u001B[0m     \u001B[38;5;66;03m# change the default number of processes to -1\u001B[39;00m\n\u001B[0;32m    262\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m parallel_backend(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend, n_jobs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_n_jobs):\n\u001B[1;32m--> 263\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m [func(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    264\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m func, args, kwargs \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mitems]\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\joblib\\parallel.py:263\u001B[0m, in \u001B[0;36m<listcomp>\u001B[1;34m(.0)\u001B[0m\n\u001B[0;32m    259\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m    260\u001B[0m     \u001B[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001B[39;00m\n\u001B[0;32m    261\u001B[0m     \u001B[38;5;66;03m# change the default number of processes to -1\u001B[39;00m\n\u001B[0;32m    262\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m parallel_backend(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend, n_jobs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_n_jobs):\n\u001B[1;32m--> 263\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m [func(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    264\u001B[0m                 \u001B[38;5;28;01mfor\u001B[39;00m func, args, kwargs \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mitems]\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\utils\\fixes.py:117\u001B[0m, in \u001B[0;36m_FuncWrapper.__call__\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m    115\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__call__\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m    116\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m config_context(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mconfig):\n\u001B[1;32m--> 117\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfunction(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\ensemble\\_forest.py:189\u001B[0m, in \u001B[0;36m_parallel_build_trees\u001B[1;34m(tree, bootstrap, X, y, sample_weight, tree_idx, n_trees, verbose, class_weight, n_samples_bootstrap)\u001B[0m\n\u001B[0;32m    186\u001B[0m     \u001B[38;5;28;01melif\u001B[39;00m class_weight \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbalanced_subsample\u001B[39m\u001B[38;5;124m\"\u001B[39m:\n\u001B[0;32m    187\u001B[0m         curr_sample_weight \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m=\u001B[39m compute_sample_weight(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbalanced\u001B[39m\u001B[38;5;124m\"\u001B[39m, y, indices\u001B[38;5;241m=\u001B[39mindices)\n\u001B[1;32m--> 189\u001B[0m     \u001B[43mtree\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msample_weight\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcurr_sample_weight\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcheck_input\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[0;32m    190\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    191\u001B[0m     tree\u001B[38;5;241m.\u001B[39mfit(X, y, sample_weight\u001B[38;5;241m=\u001B[39msample_weight, check_input\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\tree\\_classes.py:969\u001B[0m, in \u001B[0;36mDecisionTreeClassifier.fit\u001B[1;34m(self, X, y, sample_weight, check_input)\u001B[0m\n\u001B[0;32m    939\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mfit\u001B[39m(\u001B[38;5;28mself\u001B[39m, X, y, sample_weight\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, check_input\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m):\n\u001B[0;32m    940\u001B[0m     \u001B[38;5;124;03m\"\"\"Build a decision tree classifier from the training set (X, y).\u001B[39;00m\n\u001B[0;32m    941\u001B[0m \n\u001B[0;32m    942\u001B[0m \u001B[38;5;124;03m    Parameters\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    966\u001B[0m \u001B[38;5;124;03m        Fitted estimator.\u001B[39;00m\n\u001B[0;32m    967\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[1;32m--> 969\u001B[0m     \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    970\u001B[0m \u001B[43m        \u001B[49m\u001B[43mX\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    971\u001B[0m \u001B[43m        \u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    972\u001B[0m \u001B[43m        \u001B[49m\u001B[43msample_weight\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msample_weight\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    973\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcheck_input\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcheck_input\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    974\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    975\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\tree\\_classes.py:210\u001B[0m, in \u001B[0;36mBaseDecisionTree.fit\u001B[1;34m(self, X, y, sample_weight, check_input)\u001B[0m\n\u001B[0;32m    207\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mn_outputs_ \u001B[38;5;241m=\u001B[39m y\u001B[38;5;241m.\u001B[39mshape[\u001B[38;5;241m1\u001B[39m]\n\u001B[0;32m    209\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_classification:\n\u001B[1;32m--> 210\u001B[0m     \u001B[43mcheck_classification_targets\u001B[49m\u001B[43m(\u001B[49m\u001B[43my\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    211\u001B[0m     y \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39mcopy(y)\n\u001B[0;32m    213\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclasses_ \u001B[38;5;241m=\u001B[39m []\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\utils\\multiclass.py:192\u001B[0m, in \u001B[0;36mcheck_classification_targets\u001B[1;34m(y)\u001B[0m\n\u001B[0;32m    180\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcheck_classification_targets\u001B[39m(y):\n\u001B[0;32m    181\u001B[0m     \u001B[38;5;124;03m\"\"\"Ensure that target y is of a non-regression type.\u001B[39;00m\n\u001B[0;32m    182\u001B[0m \n\u001B[0;32m    183\u001B[0m \u001B[38;5;124;03m    Only the following target types (as defined in type_of_target) are allowed:\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    190\u001B[0m \u001B[38;5;124;03m        Target values.\u001B[39;00m\n\u001B[0;32m    191\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[1;32m--> 192\u001B[0m     y_type \u001B[38;5;241m=\u001B[39m \u001B[43mtype_of_target\u001B[49m\u001B[43m(\u001B[49m\u001B[43my\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43minput_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43my\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[0;32m    193\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m y_type \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m [\n\u001B[0;32m    194\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbinary\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m    195\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmulticlass\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    198\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmultilabel-sequences\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m    199\u001B[0m     ]:\n\u001B[0;32m    200\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUnknown label type: \u001B[39m\u001B[38;5;132;01m%r\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m y_type)\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\sklearn\\utils\\multiclass.py:335\u001B[0m, in \u001B[0;36mtype_of_target\u001B[1;34m(y, input_name)\u001B[0m\n\u001B[0;32m    332\u001B[0m     _assert_all_finite(y, input_name\u001B[38;5;241m=\u001B[39minput_name)\n\u001B[0;32m    333\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcontinuous\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m+\u001B[39m suffix\n\u001B[1;32m--> 335\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\u001B[38;5;28mlen\u001B[39m(\u001B[43mnp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43munique\u001B[49m\u001B[43m(\u001B[49m\u001B[43my\u001B[49m\u001B[43m)\u001B[49m) \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m2\u001B[39m) \u001B[38;5;129;01mor\u001B[39;00m (y\u001B[38;5;241m.\u001B[39mndim \u001B[38;5;241m>\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m2\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(y[\u001B[38;5;241m0\u001B[39m]) \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m):\n\u001B[0;32m    336\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmulticlass\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m+\u001B[39m suffix  \u001B[38;5;66;03m# [1, 2, 3] or [[1., 2., 3]] or [[1, 2]]\u001B[39;00m\n\u001B[0;32m    337\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n",
      "File \u001B[1;32m<__array_function__ internals>:180\u001B[0m, in \u001B[0;36munique\u001B[1;34m(*args, **kwargs)\u001B[0m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\numpy\\lib\\arraysetops.py:274\u001B[0m, in \u001B[0;36munique\u001B[1;34m(ar, return_index, return_inverse, return_counts, axis, equal_nan)\u001B[0m\n\u001B[0;32m    272\u001B[0m ar \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39masanyarray(ar)\n\u001B[0;32m    273\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m axis \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m--> 274\u001B[0m     ret \u001B[38;5;241m=\u001B[39m \u001B[43m_unique1d\u001B[49m\u001B[43m(\u001B[49m\u001B[43mar\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreturn_index\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreturn_inverse\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreturn_counts\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\n\u001B[0;32m    275\u001B[0m \u001B[43m                    \u001B[49m\u001B[43mequal_nan\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mequal_nan\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    276\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m _unpack_tuple(ret)\n\u001B[0;32m    278\u001B[0m \u001B[38;5;66;03m# axis was specified and not None\u001B[39;00m\n",
      "File \u001B[1;32m~\\anaconda3\\envs\\tf-sklearn-keras\\lib\\site-packages\\numpy\\lib\\arraysetops.py:336\u001B[0m, in \u001B[0;36m_unique1d\u001B[1;34m(ar, return_index, return_inverse, return_counts, equal_nan)\u001B[0m\n\u001B[0;32m    334\u001B[0m     aux \u001B[38;5;241m=\u001B[39m ar[perm]\n\u001B[0;32m    335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 336\u001B[0m     \u001B[43mar\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msort\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    337\u001B[0m     aux \u001B[38;5;241m=\u001B[39m ar\n\u001B[0;32m    338\u001B[0m mask \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39mempty(aux\u001B[38;5;241m.\u001B[39mshape, dtype\u001B[38;5;241m=\u001B[39mnp\u001B[38;5;241m.\u001B[39mbool_)\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "scores_strat_group_k_fold = [None]*2\n",
    "scores_strat_k_fold = [None]*2\n",
    "scores_loso = [None]*2\n",
    "\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings('ignore')\n",
    "\n",
    "    for variable in (0, 1): # phF, MF\n",
    "        model = RandomForest(path, variable=variable)\n",
    "\n",
    "        scores_strat_group_k_fold[variable] = stratified_group_k_fold(path=path,\n",
    "                                                            groups=subjects,\n",
    "                                                            model=model,\n",
    "                                                            folds=5,\n",
    "                                                            images=False,\n",
    "                                                            verbose=True,\n",
    "                                                            variable=variable)\n",
    "\n",
    "        scores_strat_k_fold[variable] = stratified_k_fold(path=path,\n",
    "                                                model=model,\n",
    "                                                folds=5,\n",
    "                                                images=False,\n",
    "                                                verbose=True,\n",
    "                                                variable=variable)\n",
    "\n",
    "        scores_loso[variable] = leave_one_subject_out(path=path,\n",
    "                                            groups=subjects,\n",
    "                                            model=model,\n",
    "                                            images=False,\n",
    "                                            verbose=True,\n",
    "                                            variable=variable)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Save scores"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "path_scores = './Scores'\n",
    "model_name = 'random_forest'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "# stratified 5-fold\n",
    "with open(f'{path_scores}/strat_5_fold/{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_strat_group_k_fold))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "# stratified group 5-fold\n",
    "with open(f'{path_scores}/strat_group_5_fold/{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_strat_k_fold))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "# LOSO\n",
    "with open(f'{path_scores}/loso/{model_name}.txt', 'w') as dat:\n",
    "    dat.write(str(scores_loso))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}