From 91fcd9e13c79f4a4697bf378efc981d3a5da34f1 Mon Sep 17 00:00:00 2001 From: shubham-s-agarwal <66172189+shubham-s-agarwal@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:10:56 +0100 Subject: [PATCH] Pushing base and advanced notebook setup --- .../meta_annotation_training.ipynb | 2481 +-------------- .../meta_annotation_training_advanced.ipynb | 2722 +++++++++++++++++ ...nnotation_training_with_oversampling.ipynb | 80 - 3 files changed, 2737 insertions(+), 2546 deletions(-) create mode 100644 medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb delete mode 100644 medcat/2_train_model/2_supervised_training/meta_annotation_training_with_oversampling.ipynb diff --git a/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb b/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb index 635c146..1f8ca89 100644 --- a/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb +++ b/medcat/2_train_model/2_supervised_training/meta_annotation_training.ipynb @@ -96,385 +96,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "f5a71c31", + "execution_count": null, + "id": "9e1720aa", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:medcat.meta_cat:LSTM model used for classification\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1684\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.46 0.40 0.43 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.92 0.94 0.93 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.46 0.45 0.45 7188\n", - "weighted avg 0.86 0.87 0.87 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.68 0.67 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.95 0.96 0.95 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.54 0.55 0.54 1797\n", - "weighted avg 0.91 0.92 0.92 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 0 and macro avg/f1-score: 0.5424475654733077 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.80 0.73 793\n", - " 1 0.61 0.17 0.27 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.64 0.65 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.76 0.72 209\n", - " 1 0.36 0.36 0.36 11\n", - " 2 0.96 0.95 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.67 0.69 0.68 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 1 and macro avg/f1-score: 0.6809929509005487 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.73 0.88 0.79 793\n", - " 1 0.47 0.44 0.46 64\n", - " 2 0.98 0.95 0.97 6331\n", - "\n", - " accuracy 0.94 7188\n", - " macro avg 0.73 0.76 0.74 7188\n", - "weighted avg 0.95 0.94 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.81 0.76 209\n", - " 1 0.31 0.45 0.37 11\n", - " 2 0.97 0.95 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.67 0.74 0.70 1797\n", - "weighted avg 0.94 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 2 and macro avg/f1-score: 0.6964827596275013 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.91 0.85 793\n", - " 1 0.62 0.73 0.67 64\n", - " 2 0.99 0.97 0.98 6331\n", - "\n", - " accuracy 0.96 7188\n", - " macro avg 0.80 0.87 0.83 7188\n", - "weighted avg 0.96 0.96 0.96 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.75 0.78 0.76 209\n", - " 1 0.31 0.36 0.33 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.68 0.70 0.69 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.84 0.92 0.88 793\n", - " 1 0.73 0.91 0.81 64\n", - " 2 0.99 0.98 0.98 6331\n", - "\n", - " accuracy 0.97 7188\n", - " macro avg 0.86 0.94 0.89 7188\n", - "weighted avg 0.97 0.97 0.97 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.75 0.78 0.77 209\n", - " 1 0.27 0.36 0.31 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.66 0.70 0.68 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.88 0.94 0.91 793\n", - " 1 0.81 0.92 0.86 64\n", - " 2 0.99 0.98 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.89 0.95 0.92 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.76 0.76 209\n", - " 1 0.29 0.36 0.32 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.67 0.69 0.68 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.89 0.94 0.91 793\n", - " 1 0.85 0.95 0.90 64\n", - " 2 0.99 0.98 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.91 0.96 0.93 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.76 0.78 209\n", - " 1 0.44 0.36 0.40 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.74 0.70 0.72 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 6 and macro avg/f1-score: 0.7163631972056036 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.90 0.94 0.92 793\n", - " 1 0.91 0.97 0.94 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.94 0.97 0.95 7188\n", - "weighted avg 0.98 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.76 0.78 209\n", - " 1 0.44 0.36 0.40 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.74 0.70 0.72 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.91 0.96 0.94 793\n", - " 1 0.94 0.98 0.96 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.98 7188\n", - " macro avg 0.95 0.98 0.96 7188\n", - "weighted avg 0.99 0.98 0.98 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.78 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.70 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.96 0.94 793\n", - " 1 0.93 0.98 0.95 64\n", - " 2 0.99 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.95 0.98 0.96 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.81 0.79 209\n", - " 1 0.38 0.45 0.42 11\n", - " 2 0.97 0.96 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.71 0.74 0.73 1797\n", - "weighted avg 0.95 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to dummy_test/model.dat at epoch: 9 and macro avg/f1-score: 0.7253847166674895 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.97 0.94 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.79 0.79 209\n", - " 1 0.33 0.45 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.69 0.74 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.92 0.97 0.94 793\n", - " 1 0.94 1.00 0.97 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.95 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.77 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.70 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.94 0.97 0.95 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.97 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.76 0.79 209\n", - " 1 0.36 0.36 0.36 11\n", - " 2 0.97 0.97 0.97 1577\n", - "\n", - " accuracy 0.95 1797\n", - " macro avg 0.71 0.70 0.71 1797\n", - "weighted avg 0.94 0.95 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.94 0.98 0.96 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.98 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.82 0.74 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.96 0.98 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.73 0.69 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.93 0.98 0.95 793\n", - " 1 0.96 1.00 0.98 64\n", - " 2 1.00 0.99 0.99 6331\n", - "\n", - " accuracy 0.99 7188\n", - " macro avg 0.96 0.99 0.98 7188\n", - "weighted avg 0.99 0.99 0.99 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.75 0.78 209\n", - " 1 0.40 0.36 0.38 11\n", - " 2 0.96 0.97 0.97 1577\n", - "\n", - " accuracy 0.94 1797\n", - " macro avg 0.72 0.69 0.71 1797\n", - "weighted avg 0.94 0.94 0.94 1797\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for meta_model in meta_model_names:\n", " vocab_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-vocab.json')\n", @@ -492,31 +117,12 @@ " \n", " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", " #Ideally this should replace the meta_models inside the modelpack\n", - " \n", - " #Below are the config values used for Experiencer classification task\n", - " \n", - " config['train']['class_weights'] = [0.4,1.5,0.1]\n", - " config.train['test_size'] = 0.2\n", - " config.train['nepochs'] = 15\n", - "\n", - " config.train.metric['base'] = 'macro avg'\n", - " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", - " \n", "\n", " # Initialise and train meta_model\n", " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" ] }, - { - "cell_type": "markdown", - "id": "0010d730", - "metadata": {}, - "source": [ - "LSTM has high weighted F1-score (0.94), however due to the class imbalance, that can be misleading.
The recall values for the minority classes (Other and Family) are low, especially for Family - 0.36\n", - "
There is room for improvement in performance, especially for the minority classes" - ] - }, { "cell_type": "markdown", "id": "91ff4e28", @@ -527,480 +133,10 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "521479e4", + "execution_count": null, + "id": "e255dda2", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1078\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.12 0.25 0.16 793\n", - " 1 0.00 0.03 0.01 64\n", - " 2 0.88 0.68 0.77 6331\n", - "\n", - " accuracy 0.63 7188\n", - " macro avg 0.34 0.32 0.31 7188\n", - "weighted avg 0.79 0.63 0.70 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.00 0.00 0.00 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.88 1.00 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.29 0.33 0.31 1797\n", - "weighted avg 0.77 0.88 0.82 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.31159849832049 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.14 0.03 0.05 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.88 0.98 0.93 6331\n", - "\n", - " accuracy 0.86 7188\n", - " macro avg 0.34 0.34 0.33 7188\n", - "weighted avg 0.79 0.86 0.82 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.00 0.00 0.00 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.88 1.00 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.29 0.33 0.31 1797\n", - "weighted avg 0.77 0.88 0.82 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.26 0.06 0.10 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.88 0.98 0.93 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.38 0.35 0.34 7188\n", - "weighted avg 0.81 0.87 0.83 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.89 0.08 0.15 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.89 1.00 0.94 1577\n", - "\n", - " accuracy 0.89 1797\n", - " macro avg 0.59 0.36 0.36 1797\n", - "weighted avg 0.88 0.89 0.84 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.36267332514096967 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.42 0.49 793\n", - " 1 0.00 0.00 0.00 64\n", - " 2 0.92 0.96 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.50 0.46 0.48 7188\n", - "weighted avg 0.88 0.90 0.88 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.61 0.68 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.94 0.98 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.57 0.53 0.55 1797\n", - "weighted avg 0.92 0.93 0.92 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 3 and macro avg/f1-score: 0.5474452879879955 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.61 0.66 0.63 793\n", - " 1 0.25 0.02 0.03 64\n", - " 2 0.95 0.95 0.95 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.60 0.54 0.54 7188\n", - "weighted avg 0.91 0.91 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.68 0.65 209\n", - " 1 0.00 0.00 0.00 11\n", - " 2 0.96 0.95 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.52 0.54 0.53 1797\n", - "weighted avg 0.91 0.91 0.91 1797\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.70 0.66 793\n", - " 1 0.68 0.23 0.35 64\n", - " 2 0.96 0.95 0.96 6331\n", - "\n", - " accuracy 0.92 7188\n", - " macro avg 0.75 0.63 0.65 7188\n", - "weighted avg 0.92 0.92 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.71 0.67 209\n", - " 1 0.45 0.45 0.45 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.68 0.70 0.69 1797\n", - "weighted avg 0.92 0.91 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 5 and macro avg/f1-score: 0.6901427504420462 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.71 0.69 793\n", - " 1 0.60 0.56 0.58 64\n", - " 2 0.96 0.95 0.96 6331\n", - "\n", - " accuracy 0.92 7188\n", - " macro avg 0.74 0.74 0.74 7188\n", - "weighted avg 0.93 0.92 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.71 0.68 209\n", - " 1 0.39 0.64 0.48 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.66 0.76 0.70 1797\n", - "weighted avg 0.92 0.91 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 6 and macro avg/f1-score: 0.7039037826541276 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.72 0.70 793\n", - " 1 0.52 0.53 0.53 64\n", - " 2 0.96 0.96 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.72 0.74 0.73 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.63 0.73 0.68 209\n", - " 1 0.41 0.64 0.50 11\n", - " 2 0.96 0.94 0.95 1577\n", - "\n", - " accuracy 0.91 1797\n", - " macro avg 0.67 0.77 0.71 1797\n", - "weighted avg 0.92 0.91 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 7 and macro avg/f1-score: 0.7085684968468765 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.75 0.71 793\n", - " 1 0.62 0.53 0.57 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.74 0.75 7188\n", - "weighted avg 0.93 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.72 0.69 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.96 0.95 0.95 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.68 0.80 0.72 1797\n", - "weighted avg 0.92 0.92 0.92 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 8 and macro avg/f1-score: 0.7214487000648947 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.77 0.73 793\n", - " 1 0.58 0.66 0.62 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.79 0.77 7188\n", - "weighted avg 0.94 0.93 0.93 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.74 0.71 209\n", - " 1 0.44 0.73 0.55 11\n", - " 2 0.96 0.95 0.96 1577\n", - "\n", - " accuracy 0.92 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.92 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 9 and macro avg/f1-score: 0.7411771369414074 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.79 0.74 793\n", - " 1 0.65 0.72 0.68 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.77 0.82 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.74 0.73 209\n", - " 1 0.42 0.73 0.53 11\n", - " 2 0.96 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.79 0.74 793\n", - " 1 0.63 0.72 0.67 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.77 0.82 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.75 0.74 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.93 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.81 0.74 793\n", - " 1 0.60 0.66 0.63 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.81 0.78 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.75 0.74 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.81 0.74 1797\n", - "weighted avg 0.94 0.93 0.93 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.80 0.75 793\n", - " 1 0.60 0.78 0.68 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.76 0.84 0.79 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.77 0.75 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.82 0.74 1797\n", - "weighted avg 0.94 0.93 0.94 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 13 and macro avg/f1-score: 0.7440346627353422 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.80 0.75 793\n", - " 1 0.56 0.67 0.61 64\n", - " 2 0.97 0.95 0.96 6331\n", - "\n", - " accuracy 0.93 7188\n", - " macro avg 0.75 0.81 0.77 7188\n", - "weighted avg 0.94 0.93 0.94 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.76 0.75 209\n", - " 1 0.40 0.73 0.52 11\n", - " 2 0.97 0.96 0.96 1577\n", - "\n", - " accuracy 0.93 1797\n", - " macro avg 0.70 0.82 0.74 1797\n", - "weighted avg 0.94 0.93 0.94 1797\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for meta_model in meta_model_names:\n", " # load and sort out the config\n", @@ -1011,53 +147,18 @@ " for key, value in config_dict.items():\n", " setattr(config, key, value['py/state']['__dict__'])\n", "\n", + " tokenizer = TokenizerWrapperBERT.load(os.path.join(base_dir_meta_models,\"meta_\"+meta_model), \n", + " config.model['model_variant'])\n", + " \n", " # change model name if training BERT for the first time\n", - " config['model']['model_name'] = 'bert'\n", - "\n", - " # change input_size as well\n", - " config['model']['input_size'] = 768\n", - "\n", - " # you can also switch between freezing BERT layers or using LoRA during training\n", - " # to use LORA:\n", - " config.model['model_freeze_layers'] = False\n", - "\n", - " tokenizer = TokenizerWrapperBERT.load(os.path.join(base_dir_meta_models,\"meta_\"+meta_model), config['model']['model_variant'])\n", + " config.model['model_name'] = 'bert'\n", " \n", " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", " #Ideally this should replace the meta_models inside the modelpack\n", "\n", - " #Below are the config values used for Experiencer classification task\n", - "\n", - " config['model']['nclasses'] = 3\n", - " config['general']['category_name'] = 'Experiencer'\n", - "\n", - " config.train.lr = 5e-4\n", - " config.train['test_size'] = 0.2\n", - " config['train']['nepochs'] = 15\n", - " config.model['model_freeze_layers'] = False\n", - "\n", - " config.train.metric['base'] = 'macro avg'\n", - " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", - "\n", - " config['train']['class_weights'] = [0.4,1.5,0.1]\n", - "\n", - " save_dir_path= \"test_meta\" # Where to save the meta_model and results.\n", - " #Ideally this should replace the meta_models inside the modelpack\n", - "\n", " # Initialise and train meta_model\n", " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", - "\n", - " # Save results\n", - " json.dump(results, open(os.path.join(save_dir_path,'meta_results.json'), 'w'))" - ] - }, - { - "cell_type": "markdown", - "id": "8dc9496b", - "metadata": {}, - "source": [ - "BERT model shows improvement for the Family class compared to LSTM, however the recall values can still be improved.
To help tackle this, we'll use 2 phase learning for training." + " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" ] }, { @@ -1077,1573 +178,21 @@ "source": [ "config = ConfigMetaCAT()\n", "# make sure to change the following parameters:\n", - "# config['model']['nclasses']\n", - "# config['general']['category_name']\n", + "# config.model['nclasses']\n", + "# config.general['category_name']\n", "\n", "# change model name if training BERT for the first time\n", - "config['model']['model_name'] = 'bert'\n", + "config.model['model_name'] = 'bert'\n", "\n", - "tokenizer = TokenizerWrapperBERT.load(\"\", config['model']['model_variant'])\n", + "tokenizer = TokenizerWrapperBERT.load(\"\", config.model['model_variant'])\n", "\n", "save_dir_path= \"test_meta\" # Where to save the meta_model and results. \n", "#Ideally this should replace the meta_models inside the modelpack\n", "\n", "# Initialise and train meta_model\n", "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", - "\n", - "# Save results\n", - "json.dump(results, open(os.path.join(save_dir_path,'meta_results.json'), 'w'))" - ] - }, - { - "cell_type": "markdown", - "id": "c3002ef0", - "metadata": {}, - "source": [ - "## If using 2 phase learning for training" - ] - }, - { - "cell_type": "markdown", - "id": "b83d9958", - "metadata": {}, - "source": [ - "### Phase 1" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9bcdf7be", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 665\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.80 0.60 809\n", - " 1 0.03 0.03 0.03 61\n", - " 2 0.44 0.13 0.20 793\n", - "\n", - " accuracy 0.45 1663\n", - " macro avg 0.32 0.32 0.27 1663\n", - "weighted avg 0.44 0.45 0.39 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 1.00 0.63 193\n", - " 1 0.00 0.00 0.00 14\n", - " 2 1.00 0.00 0.01 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.49 0.33 0.21 416\n", - "weighted avg 0.72 0.47 0.30 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.21479741019214701 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.85 0.62 809\n", - " 1 0.06 0.08 0.07 61\n", - " 2 0.44 0.09 0.14 793\n", - "\n", - " accuracy 0.46 1663\n", - " macro avg 0.33 0.34 0.28 1663\n", - "weighted avg 0.45 0.46 0.37 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 1.00 0.63 193\n", - " 1 1.00 0.07 0.13 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.49 0.36 0.26 416\n", - "weighted avg 0.25 0.47 0.30 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 1 and macro avg/f1-score: 0.2560672514619883 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.50 0.89 0.64 809\n", - " 1 0.14 0.36 0.20 61\n", - " 2 0.58 0.05 0.09 793\n", - "\n", - " accuracy 0.47 1663\n", - " macro avg 0.41 0.43 0.31 1663\n", - "weighted avg 0.53 0.47 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.47 0.97 0.63 193\n", - " 1 0.31 0.36 0.33 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.26 0.44 0.32 416\n", - "weighted avg 0.23 0.46 0.31 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.322465804759228 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.90 0.67 809\n", - " 1 0.15 0.66 0.24 61\n", - " 2 0.41 0.01 0.02 793\n", - "\n", - " accuracy 0.47 1663\n", - " macro avg 0.36 0.52 0.31 1663\n", - "weighted avg 0.46 0.47 0.34 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.90 0.71 193\n", - " 1 0.09 0.79 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.44 416\n", - " macro avg 0.23 0.56 0.29 416\n", - "weighted avg 0.27 0.44 0.33 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 809\n", - " 1 0.15 0.80 0.26 61\n", - " 2 0.75 0.01 0.01 793\n", - "\n", - " accuracy 0.48 1663\n", - " macro avg 0.49 0.58 0.32 1663\n", - "weighted avg 0.64 0.48 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.60 0.96 0.74 193\n", - " 1 0.08 0.64 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.23 0.53 0.30 416\n", - "weighted avg 0.28 0.47 0.35 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.96 0.73 809\n", - " 1 0.15 0.82 0.26 61\n", - " 2 1.00 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.58 0.59 0.33 1663\n", - "weighted avg 0.77 0.50 0.36 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.94 0.77 193\n", - " 1 0.07 0.71 0.13 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.24 0.55 0.30 416\n", - "weighted avg 0.31 0.46 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.63 0.94 0.75 809\n", - " 1 0.13 0.90 0.22 61\n", - " 2 1.00 0.01 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.58 0.62 0.33 1663\n", - "weighted avg 0.79 0.50 0.38 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.95 0.77 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.30 0.47 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.64 0.95 0.76 809\n", - " 1 0.12 0.93 0.22 61\n", - " 2 0.75 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.50 0.63 0.33 1663\n", - "weighted avg 0.67 0.50 0.38 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.65 0.94 0.77 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.46 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.46 0.36 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.95 0.78 809\n", - " 1 0.12 0.95 0.21 61\n", - " 2 1.00 0.01 0.02 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.59 0.64 0.34 1663\n", - "weighted avg 0.81 0.50 0.40 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.95 0.78 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.47 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.95 0.78 809\n", - " 1 0.12 0.95 0.21 61\n", - " 2 1.00 0.00 0.01 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.59 0.64 0.33 1663\n", - "weighted avg 0.80 0.50 0.39 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.94 0.78 193\n", - " 1 0.08 0.86 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.47 416\n", - " macro avg 0.25 0.60 0.31 416\n", - "weighted avg 0.31 0.47 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.95 0.79 809\n", - " 1 0.12 0.97 0.21 61\n", - " 2 1.00 0.01 0.02 793\n", - "\n", - " accuracy 0.51 1663\n", - " macro avg 0.60 0.64 0.34 1663\n", - "weighted avg 0.81 0.51 0.40 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.97 0.80 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 10 and macro avg/f1-score: 0.32250773257197235 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.95 0.80 809\n", - " 1 0.11 0.97 0.20 61\n", - " 2 0.77 0.01 0.02 793\n", - "\n", - " accuracy 0.50 1663\n", - " macro avg 0.52 0.64 0.34 1663\n", - "weighted avg 0.71 0.50 0.41 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.97 0.79 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.31 0.48 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.96 0.80 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 1.00 0.02 0.03 793\n", - "\n", - " accuracy 0.51 1663\n", - " macro avg 0.60 0.66 0.35 1663\n", - "weighted avg 0.81 0.51 0.41 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.97 0.80 193\n", - " 1 0.09 0.93 0.17 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.63 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 12 and macro avg/f1-score: 0.3237202463008914 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.96 0.80 809\n", - " 1 0.12 0.98 0.22 61\n", - " 2 0.96 0.03 0.06 793\n", - "\n", - " accuracy 0.52 1663\n", - " macro avg 0.59 0.66 0.36 1663\n", - "weighted avg 0.79 0.52 0.42 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.98 0.79 193\n", - " 1 0.09 0.79 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.25 0.59 0.31 416\n", - "weighted avg 0.31 0.48 0.37 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.21 61\n", - " 2 0.88 0.03 0.06 793\n", - "\n", - " accuracy 0.52 1663\n", - " macro avg 0.57 0.67 0.36 1663\n", - "weighted avg 0.77 0.52 0.43 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.08 0.86 0.15 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.61 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.97 0.82 809\n", - " 1 0.12 1.00 0.21 61\n", - " 2 0.95 0.05 0.10 793\n", - "\n", - " accuracy 0.53 1663\n", - " macro avg 0.59 0.67 0.38 1663\n", - "weighted avg 0.80 0.53 0.45 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Test\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.81 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 0.00 0.00 0.00 209\n", - "\n", - " accuracy 0.48 416\n", - " macro avg 0.26 0.61 0.32 416\n", - "weighted avg 0.32 0.48 0.38 416\n", - "\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 0.94 0.04 0.07 793\n", - "\n", - " accuracy 0.53 1663\n", - " macro avg 0.58 0.67 0.37 1663\n", - "weighted avg 0.79 0.53 0.44 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.80 193\n", - " 1 0.09 0.86 0.16 14\n", - " 2 1.00 0.00 0.01 209\n", - "\n", - " accuracy 0.49 416\n", - " macro avg 0.59 0.61 0.32 416\n", - "weighted avg 0.82 0.49 0.38 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.97 0.81 809\n", - " 1 0.12 1.00 0.22 61\n", - " 2 0.94 0.06 0.12 793\n", - "\n", - " accuracy 0.54 1663\n", - " macro avg 0.59 0.68 0.39 1663\n", - "weighted avg 0.80 0.54 0.46 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.66 0.98 0.79 193\n", - " 1 0.09 0.86 0.17 14\n", - " 2 1.00 0.02 0.04 209\n", - "\n", - " accuracy 0.49 416\n", - " macro avg 0.59 0.62 0.33 416\n", - "weighted avg 0.81 0.49 0.39 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 17 and macro avg/f1-score: 0.3328554768276514 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 809\n", - " 1 0.14 1.00 0.24 61\n", - " 2 0.95 0.08 0.15 793\n", - "\n", - " accuracy 0.55 1663\n", - " macro avg 0.59 0.69 0.40 1663\n", - "weighted avg 0.80 0.55 0.47 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.67 0.98 0.80 193\n", - " 1 0.09 0.86 0.17 14\n", - " 2 1.00 0.04 0.07 209\n", - "\n", - " accuracy 0.50 416\n", - " macro avg 0.59 0.62 0.35 416\n", - "weighted avg 0.82 0.50 0.41 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 18 and macro avg/f1-score: 0.3471379464272786 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.98 0.82 809\n", - " 1 0.13 1.00 0.24 61\n", - " 2 0.97 0.09 0.17 793\n", - "\n", - " accuracy 0.56 1663\n", - " macro avg 0.60 0.69 0.41 1663\n", - "weighted avg 0.81 0.56 0.49 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.80 193\n", - " 1 0.10 0.86 0.17 14\n", - " 2 1.00 0.07 0.13 209\n", - "\n", - " accuracy 0.52 416\n", - " macro avg 0.59 0.63 0.37 416\n", - "weighted avg 0.82 0.52 0.44 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 19 and macro avg/f1-score: 0.3674925759230418 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.98 0.83 809\n", - " 1 0.13 1.00 0.23 61\n", - " 2 0.94 0.10 0.18 793\n", - "\n", - " accuracy 0.56 1663\n", - " macro avg 0.59 0.69 0.41 1663\n", - "weighted avg 0.80 0.56 0.49 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.10 0.86 0.18 14\n", - " 2 1.00 0.10 0.17 209\n", - "\n", - " accuracy 0.53 416\n", - " macro avg 0.60 0.64 0.39 416\n", - "weighted avg 0.83 0.53 0.47 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 20 and macro avg/f1-score: 0.38685497295333987 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.98 0.81 809\n", - " 1 0.14 1.00 0.25 61\n", - " 2 0.96 0.12 0.22 793\n", - "\n", - " accuracy 0.57 1663\n", - " macro avg 0.60 0.70 0.43 1663\n", - "weighted avg 0.80 0.57 0.51 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.68 0.98 0.81 193\n", - " 1 0.11 0.86 0.19 14\n", - " 2 0.96 0.11 0.21 209\n", - "\n", - " accuracy 0.54 416\n", - " macro avg 0.58 0.65 0.40 416\n", - "weighted avg 0.80 0.54 0.48 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 21 and macro avg/f1-score: 0.4002297137890358 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.70 0.99 0.82 809\n", - " 1 0.14 1.00 0.25 61\n", - " 2 0.98 0.12 0.22 793\n", - "\n", - " accuracy 0.57 1663\n", - " macro avg 0.61 0.70 0.43 1663\n", - "weighted avg 0.81 0.57 0.51 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.98 0.81 193\n", - " 1 0.11 0.86 0.20 14\n", - " 2 0.94 0.15 0.26 209\n", - "\n", - " accuracy 0.56 416\n", - " macro avg 0.58 0.66 0.42 416\n", - "weighted avg 0.80 0.56 0.51 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 22 and macro avg/f1-score: 0.4231725462046246 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.99 0.83 809\n", - " 1 0.15 1.00 0.26 61\n", - " 2 0.99 0.17 0.29 793\n", - "\n", - " accuracy 0.60 1663\n", - " macro avg 0.62 0.72 0.46 1663\n", - "weighted avg 0.82 0.60 0.55 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.69 0.99 0.81 193\n", - " 1 0.12 0.86 0.21 14\n", - " 2 1.00 0.19 0.31 209\n", - "\n", - " accuracy 0.58 416\n", - " macro avg 0.60 0.68 0.45 416\n", - "weighted avg 0.83 0.58 0.54 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 23 and macro avg/f1-score: 0.4459035717026967 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.99 0.83 809\n", - " 1 0.16 1.00 0.28 61\n", - " 2 0.99 0.21 0.35 793\n", - "\n", - " accuracy 0.62 1663\n", - " macro avg 0.62 0.73 0.49 1663\n", - "weighted avg 0.83 0.62 0.58 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.98 0.83 193\n", - " 1 0.12 0.86 0.21 14\n", - " 2 0.98 0.25 0.40 209\n", - "\n", - " accuracy 0.61 416\n", - " macro avg 0.61 0.70 0.48 416\n", - "weighted avg 0.83 0.61 0.60 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 24 and macro avg/f1-score: 0.482945795402635 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.71 0.99 0.83 809\n", - " 1 0.15 1.00 0.27 61\n", - " 2 0.98 0.18 0.31 793\n", - "\n", - " accuracy 0.60 1663\n", - " macro avg 0.62 0.72 0.47 1663\n", - "weighted avg 0.82 0.60 0.56 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.98 0.83 193\n", - " 1 0.13 0.86 0.22 14\n", - " 2 0.98 0.27 0.43 209\n", - "\n", - " accuracy 0.62 416\n", - " macro avg 0.61 0.70 0.49 416\n", - "weighted avg 0.83 0.62 0.61 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 25 and macro avg/f1-score: 0.493566120394729 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.72 0.99 0.83 809\n", - " 1 0.17 1.00 0.29 61\n", - " 2 0.99 0.24 0.39 793\n", - "\n", - " accuracy 0.63 1663\n", - " macro avg 0.63 0.74 0.50 1663\n", - "weighted avg 0.83 0.63 0.60 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.84 193\n", - " 1 0.14 0.86 0.24 14\n", - " 2 0.97 0.34 0.50 209\n", - "\n", - " accuracy 0.65 416\n", - " macro avg 0.62 0.73 0.53 416\n", - "weighted avg 0.84 0.65 0.65 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 26 and macro avg/f1-score: 0.52768022857349 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.85 809\n", - " 1 0.17 1.00 0.29 61\n", - " 2 0.98 0.28 0.43 793\n", - "\n", - " accuracy 0.65 1663\n", - " macro avg 0.63 0.75 0.52 1663\n", - "weighted avg 0.84 0.65 0.63 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.98 0.84 193\n", - " 1 0.15 0.86 0.25 14\n", - " 2 0.97 0.37 0.53 209\n", - "\n", - " accuracy 0.67 416\n", - " macro avg 0.62 0.73 0.54 416\n", - "weighted avg 0.84 0.67 0.67 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 27 and macro avg/f1-score: 0.5428240740740741 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.99 0.85 809\n", - " 1 0.18 1.00 0.31 61\n", - " 2 0.98 0.31 0.48 793\n", - "\n", - " accuracy 0.67 1663\n", - " macro avg 0.64 0.77 0.54 1663\n", - "weighted avg 0.84 0.67 0.65 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.86 193\n", - " 1 0.15 0.86 0.26 14\n", - " 2 0.98 0.41 0.58 209\n", - "\n", - " accuracy 0.69 416\n", - " macro avg 0.63 0.75 0.56 416\n", - "weighted avg 0.85 0.69 0.70 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 28 and macro avg/f1-score: 0.5638621950678688 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.74 0.99 0.85 809\n", - " 1 0.19 1.00 0.32 61\n", - " 2 0.99 0.33 0.50 793\n", - "\n", - " accuracy 0.68 1663\n", - " macro avg 0.64 0.77 0.55 1663\n", - "weighted avg 0.84 0.68 0.66 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.85 193\n", - " 1 0.16 0.86 0.27 14\n", - " 2 0.98 0.43 0.59 209\n", - "\n", - " accuracy 0.70 416\n", - " macro avg 0.63 0.75 0.57 416\n", - "weighted avg 0.85 0.70 0.70 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 29 and macro avg/f1-score: 0.5720897974597215 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.99 0.86 809\n", - " 1 0.18 1.00 0.30 61\n", - " 2 0.99 0.34 0.51 793\n", - "\n", - " accuracy 0.68 1663\n", - " macro avg 0.64 0.78 0.56 1663\n", - "weighted avg 0.85 0.68 0.67 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.86 193\n", - " 1 0.17 0.86 0.29 14\n", - " 2 0.98 0.48 0.65 209\n", - "\n", - " accuracy 0.73 416\n", - " macro avg 0.64 0.77 0.60 416\n", - "weighted avg 0.86 0.73 0.74 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 30 and macro avg/f1-score: 0.6005270274314415 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.99 0.87 809\n", - " 1 0.21 1.00 0.35 61\n", - " 2 0.99 0.43 0.60 793\n", - "\n", - " accuracy 0.72 1663\n", - " macro avg 0.66 0.81 0.61 1663\n", - "weighted avg 0.86 0.72 0.72 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.86 193\n", - " 1 0.21 0.86 0.33 14\n", - " 2 0.98 0.54 0.69 209\n", - "\n", - " accuracy 0.75 416\n", - " macro avg 0.65 0.79 0.63 416\n", - "weighted avg 0.86 0.75 0.76 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 31 and macro avg/f1-score: 0.6306067812327066 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.98 0.87 809\n", - " 1 0.23 1.00 0.37 61\n", - " 2 0.98 0.46 0.63 793\n", - "\n", - " accuracy 0.73 1663\n", - " macro avg 0.66 0.81 0.62 1663\n", - "weighted avg 0.85 0.73 0.73 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.87 193\n", - " 1 0.21 0.86 0.34 14\n", - " 2 0.98 0.56 0.71 209\n", - "\n", - " accuracy 0.76 416\n", - " macro avg 0.66 0.80 0.64 416\n", - "weighted avg 0.87 0.76 0.77 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 32 and macro avg/f1-score: 0.6414740059803422 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.77 0.99 0.87 809\n", - " 1 0.23 1.00 0.38 61\n", - " 2 0.99 0.46 0.63 793\n", - "\n", - " accuracy 0.74 1663\n", - " macro avg 0.67 0.82 0.62 1663\n", - "weighted avg 0.86 0.74 0.74 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.87 193\n", - " 1 0.22 0.86 0.35 14\n", - " 2 0.98 0.57 0.72 209\n", - "\n", - " accuracy 0.77 416\n", - " macro avg 0.66 0.80 0.65 416\n", - "weighted avg 0.87 0.77 0.78 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 33 and macro avg/f1-score: 0.6473391409830812 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.76 0.98 0.86 809\n", - " 1 0.23 1.00 0.38 61\n", - " 2 0.99 0.45 0.62 793\n", - "\n", - " accuracy 0.73 1663\n", - " macro avg 0.66 0.81 0.62 1663\n", - "weighted avg 0.85 0.73 0.73 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.98 0.88 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.99 0.63 0.77 209\n", - "\n", - " accuracy 0.80 416\n", - " macro avg 0.68 0.82 0.69 416\n", - "weighted avg 0.87 0.80 0.81 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 34 and macro avg/f1-score: 0.6868341887176047 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.99 0.87 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.99 0.51 0.67 793\n", - "\n", - " accuracy 0.76 1663\n", - " macro avg 0.68 0.83 0.65 1663\n", - "weighted avg 0.86 0.76 0.76 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.23 0.86 0.36 14\n", - " 2 0.98 0.62 0.76 209\n", - "\n", - " accuracy 0.79 416\n", - " macro avg 0.67 0.82 0.67 416\n", - "weighted avg 0.88 0.79 0.80 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.79 0.99 0.88 809\n", - " 1 0.24 1.00 0.39 61\n", - " 2 0.99 0.50 0.66 793\n", - "\n", - " accuracy 0.75 1663\n", - " macro avg 0.67 0.83 0.64 1663\n", - "weighted avg 0.87 0.75 0.76 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.69 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 36 and macro avg/f1-score: 0.6928908001266963 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.98 0.88 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.98 0.53 0.69 793\n", - "\n", - " accuracy 0.77 1663\n", - " macro avg 0.68 0.84 0.66 1663\n", - "weighted avg 0.87 0.77 0.77 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:\n", - "##### Model saved to test_meta/model.dat at epoch: 37 and macro avg/f1-score: 0.6963856616166311 #####\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.78 0.98 0.87 809\n", - " 1 0.25 1.00 0.41 61\n", - " 2 0.98 0.50 0.66 793\n", - "\n", - " accuracy 0.75 1663\n", - " macro avg 0.67 0.83 0.65 1663\n", - "weighted avg 0.86 0.75 0.75 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.80 0.98 0.88 809\n", - " 1 0.26 1.00 0.41 61\n", - " 2 0.98 0.53 0.69 793\n", - "\n", - " accuracy 0.77 1663\n", - " macro avg 0.68 0.84 0.66 1663\n", - "weighted avg 0.86 0.77 0.77 1663\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.81 0.97 0.89 193\n", - " 1 0.27 0.86 0.41 14\n", - " 2 0.98 0.66 0.79 209\n", - "\n", - " accuracy 0.81 416\n", - " macro avg 0.69 0.83 0.70 416\n", - "weighted avg 0.88 0.81 0.82 416\n", - "\n" - ] - } - ], - "source": [ - "# Follow same steps till defining save_dir_path\n", - "\n", - "######################################################################################################\n", - "# 2 phase learning (used for imbalanced datasets) - trains the models twice: \n", - "# phase 1: trains for minority class(es) by undersampling data\n", - "# phase 2: trains for all classes\n", - "# parameter values: \n", - "# 1: Phase 1 - Train model on undersampled data\n", - "# 2: Phase 2 - Continue training on full data\n", - "# 0: None\n", - "#\n", - "# Paper reference - https://ieeexplore.ieee.org/document/7533053\n", - "# NOTE: Make sure to use class weights in favour of minority classes with 2 phase learning\n", - "#####################################################################################################\n", - "\n", - "config.model.phase_number = 1\n", - "\n", - "# specify the class that will define the desired sample size for the undersampling process\n", - "# if this is left empty, the class with the lowest samples will be chosen\n", - "# example\n", - "config.model['category_undersample'] = 'Other'\n", - "\n", - "\n", - "# To use class weights, we have 2 options:\n", - "\n", - "# 1st option:\n", - "# using specified class weights\n", - "# example with 2 classes\n", - "config['train']['class_weights'] = [0.3,0.7]\n", - "\n", - "# 2nd option:\n", - "# to calculate class weights based on class distribution\n", - "# NOTE: this will only be applicable if config.train.class_weights is empty\n", - "config['train']['compute_class_weights'] = True\n", - "\n", - "# NOTE: when using class weights, it is recommended to define the category to index mapping to ensure the weights are assigned to the right class\n", - "config['general']['category_value2id'] = {\"Other\":1,\"Confirmed\":0}\n", - "\n", - "\n", - "#Below are the config values used for Experiencer classification task\n", - "\n", - "config.model.phase_number = 1\n", - "config['model']['nclasses'] = 3\n", - "config['general']['category_name'] = 'Experiencer'\n", - "\n", - "config.train.lr = 5e-4\n", - "config.train['test_size'] = 0.2\n", - "\n", - "config.train.metric['base'] = 'macro avg'\n", - "config.general['category_value2id'] = {'Other':0, 'Family':1, 'Patient':2}\n", - "\n", - "config.model['category_undersample'] = 'Other'\n", - "config['train']['class_weights'] = [0.4,1.5,0.05]\n", - "config.model['model_freeze_layers'] = False\n", - "config['train']['nepochs'] = 20\n", - "\n", - "# Initialise and train meta_model \n", - "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" ] - }, - { - "cell_type": "markdown", - "id": "e229ef0a", - "metadata": {}, - "source": [ - "## Phase 2" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "63c01002", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", - "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", - "WARNING:medcat.utils.meta_cat.models:\n", - "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", - "INFO:medcat.meta_cat:BERT model used for classification\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", - "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", - "INFO:medcat.meta_cat:Model state loaded from dict for 2 phase learning\n", - "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - " warnings.warn(\n", - "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 718\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.32 0.94 0.48 793\n", - " 1 0.07 0.97 0.13 64\n", - " 2 0.99 0.62 0.76 6331\n", - "\n", - " accuracy 0.66 7188\n", - " macro avg 0.46 0.84 0.45 7188\n", - "weighted avg 0.91 0.66 0.72 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.40 0.95 0.56 209\n", - " 1 0.10 1.00 0.18 11\n", - " 2 0.99 0.75 0.85 1577\n", - "\n", - " accuracy 0.77 1797\n", - " macro avg 0.50 0.90 0.53 1797\n", - "weighted avg 0.92 0.77 0.81 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.42 0.92 0.58 793\n", - " 1 0.15 0.97 0.26 64\n", - " 2 0.99 0.79 0.88 6331\n", - "\n", - " accuracy 0.80 7188\n", - " macro avg 0.52 0.89 0.57 7188\n", - "weighted avg 0.92 0.80 0.84 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.49 0.92 0.64 209\n", - " 1 0.12 1.00 0.21 11\n", - " 2 0.99 0.82 0.90 1577\n", - "\n", - " accuracy 0.83 1797\n", - " macro avg 0.53 0.91 0.58 1797\n", - "weighted avg 0.92 0.83 0.86 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.48 0.92 0.63 793\n", - " 1 0.21 0.97 0.35 64\n", - " 2 0.99 0.84 0.91 6331\n", - "\n", - " accuracy 0.85 7188\n", - " macro avg 0.56 0.91 0.63 7188\n", - "weighted avg 0.93 0.85 0.87 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.49 0.94 0.64 209\n", - " 1 0.13 1.00 0.23 11\n", - " 2 0.99 0.82 0.90 1577\n", - "\n", - " accuracy 0.84 1797\n", - " macro avg 0.54 0.92 0.59 1797\n", - "weighted avg 0.93 0.84 0.87 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.92 0.67 793\n", - " 1 0.25 0.95 0.40 64\n", - " 2 0.99 0.87 0.92 6331\n", - "\n", - " accuracy 0.87 7188\n", - " macro avg 0.59 0.91 0.66 7188\n", - "weighted avg 0.93 0.87 0.89 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.55 0.92 0.69 209\n", - " 1 0.14 1.00 0.25 11\n", - " 2 0.99 0.86 0.92 1577\n", - "\n", - " accuracy 0.87 1797\n", - " macro avg 0.56 0.93 0.62 1797\n", - "weighted avg 0.93 0.87 0.89 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.53 0.93 0.68 793\n", - " 1 0.31 0.95 0.47 64\n", - " 2 0.99 0.88 0.93 6331\n", - "\n", - " accuracy 0.88 7188\n", - " macro avg 0.61 0.92 0.69 7188\n", - "weighted avg 0.93 0.88 0.90 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 209\n", - " 1 0.15 1.00 0.27 11\n", - " 2 0.99 0.87 0.92 1577\n", - "\n", - " accuracy 0.87 1797\n", - " macro avg 0.57 0.93 0.63 1797\n", - "weighted avg 0.93 0.87 0.89 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.56 0.92 0.70 793\n", - " 1 0.32 0.92 0.47 64\n", - " 2 0.99 0.89 0.94 6331\n", - "\n", - " accuracy 0.89 7188\n", - " macro avg 0.62 0.91 0.70 7188\n", - "weighted avg 0.94 0.89 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.57 0.92 0.71 209\n", - " 1 0.17 1.00 0.29 11\n", - " 2 0.99 0.87 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.58 0.93 0.64 1797\n", - "weighted avg 0.93 0.88 0.90 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.92 0.71 793\n", - " 1 0.34 0.95 0.50 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.63 0.92 0.72 7188\n", - "weighted avg 0.94 0.90 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.58 0.90 0.71 209\n", - " 1 0.16 1.00 0.28 11\n", - " 2 0.99 0.88 0.93 1577\n", - "\n", - " accuracy 0.88 1797\n", - " macro avg 0.58 0.93 0.64 1797\n", - "weighted avg 0.93 0.88 0.90 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.93 0.72 793\n", - " 1 0.32 0.97 0.48 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.90 7188\n", - " macro avg 0.63 0.93 0.71 7188\n", - "weighted avg 0.94 0.90 0.91 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.60 0.89 0.72 209\n", - " 1 0.19 1.00 0.31 11\n", - " 2 0.98 0.89 0.94 1577\n", - "\n", - " accuracy 0.89 1797\n", - " macro avg 0.59 0.93 0.66 1797\n", - "weighted avg 0.94 0.89 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.59 0.92 0.72 793\n", - " 1 0.35 0.97 0.51 64\n", - " 2 0.99 0.90 0.94 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.64 0.93 0.73 7188\n", - "weighted avg 0.94 0.91 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.89 0.73 209\n", - " 1 0.20 1.00 0.33 11\n", - " 2 0.98 0.90 0.94 1577\n", - "\n", - " accuracy 0.90 1797\n", - " macro avg 0.60 0.93 0.67 1797\n", - "weighted avg 0.94 0.90 0.91 1797\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.62 0.92 0.74 793\n", - " 1 0.36 0.95 0.52 64\n", - " 2 0.99 0.91 0.95 6331\n", - "\n", - " accuracy 0.91 7188\n", - " macro avg 0.66 0.93 0.74 7188\n", - "weighted avg 0.94 0.91 0.92 7188\n", - "\n", - "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", - "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", - "\n", - " 0 0.64 0.88 0.74 209\n", - " 1 0.21 1.00 0.34 11\n", - " 2 0.98 0.91 0.94 1577\n", - "\n", - " accuracy 0.90 1797\n", - " macro avg 0.61 0.93 0.68 1797\n", - "weighted avg 0.94 0.90 0.92 1797\n", - "\n" - ] - } - ], - "source": [ - "# Perform 2nd round of training\n", - "\n", - "config.model['phase_number'] = 2\n", - "config['train']['class_weights'] = [0.3,1,0.05]\n", - "config['train']['nepochs'] = 10\n", - "\n", - "results = mc.train_from_json(json_files, save_dir_path=save_dir_path)" - ] - }, - { - "cell_type": "markdown", - "id": "62771904", - "metadata": {}, - "source": [ - "Using 2 phase learning boosts the performance of the model, especially for the minority classes, wiht 0.88 and 1 recall values.
This highlights the impact of using 2 phase learning for model training with imbalanced datasets\n", - "

NOTE: The observed performance improvements are dataset-dependent, and you may not experience such substantial gains. Additionally, class weights and other hyperparameters will need to be fine-tuned for your specific dataset." - ] } ], "metadata": { diff --git a/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb b/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb new file mode 100644 index 0000000..4044d5b --- /dev/null +++ b/medcat/2_train_model/2_supervised_training/meta_annotation_training_advanced.ipynb @@ -0,0 +1,2722 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ae1fe3b4", + "metadata": {}, + "source": [ + "### This notebook is an advanced tutorial detailing the config changes for optimising the BERT and LSTM models for Experiencer classification task on custom dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d58c720d", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from datetime import date\n", + "from medcat.cat import CAT\n", + "from medcat.meta_cat import MetaCAT\n", + "from medcat.config_meta_cat import ConfigMetaCAT\n", + "from medcat.tokenizers.meta_cat_tokenizers import TokenizerWrapperBPE, TokenizerWrapperBERT\n", + "from tokenizers import ByteLevelBPETokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ca80af0e", + "metadata": {}, + "outputs": [], + "source": [ + "# if you want to enable info level logging\n", + "import logging\n", + "logging.basicConfig(level=logging.INFO,force=True)" + ] + }, + { + "cell_type": "markdown", + "id": "5d0606ec", + "metadata": {}, + "source": [ + "# Set parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "dd7a2e97", + "metadata": {}, + "outputs": [], + "source": [ + "# relative path to working_with_cogstack folder\n", + "_rel_path = os.path.join(\"..\", \"..\", \"..\")\n", + "# absolute path to working_with_cogstack folder\n", + "base_path = os.path.abspath(_rel_path)\n", + "# Load mct export\n", + "ann_dir = os.path.join(base_path, \"data\", \"medcattrainer_export\")\n", + "\n", + "mctrainer_export_path = ann_dir + \"\" # name of your mct export\n", + "\n", + "# Load model\n", + "model_dir = os.path.join(base_path, \"models\", \"modelpack\")\n", + "modelpack = '' # name of modelpack\n", + "model_pack_path = os.path.join(model_dir, modelpack)\n", + " #output_modelpack = model_dir + f\"{today}_trained_model\"\n", + "\n", + "# will be used to date the trained model\n", + "today = str(date.today())\n", + "today = today.replace(\"-\",\"\")\n", + "\n", + "# Initialise meta_ann models\n", + "if model_pack_path[-4:] == '.zip':\n", + " base_dir_meta_models = model_pack_path[:-4]\n", + "else:\n", + " base_dir_meta_models = model_pack_path\n", + "\n", + "# Iterate through the meta_models contained in the model\n", + "meta_model_names = [] # These Meta_annotation tasks should correspond to the ones labelled in the mcttrainer export\n", + "for dirpath, dirnames, filenames in os.walk(base_dir_meta_models):\n", + " for dirname in dirnames:\n", + " if dirname.startswith('meta_'):\n", + " meta_model_names.append(dirname[5:])" + ] + }, + { + "cell_type": "markdown", + "id": "35aa5605", + "metadata": {}, + "source": [ + "Before you run the next section please double check that the model meta_annotation names matches to those specified in the mct export.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "83701c19", + "metadata": {}, + "source": [ + "# For LSTM model" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f5a71c31", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:medcat.meta_cat:LSTM model used for classification\n", + "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n", + "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1684\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.46 0.40 0.43 793\n", + " 1 0.00 0.00 0.00 64\n", + " 2 0.92 0.94 0.93 6331\n", + "\n", + " accuracy 0.87 7188\n", + " macro avg 0.46 0.45 0.45 7188\n", + "weighted avg 0.86 0.87 0.87 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.68 0.67 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.95 0.96 0.95 1577\n", + "\n", + " accuracy 0.92 1797\n", + " macro avg 0.54 0.55 0.54 1797\n", + "weighted avg 0.91 0.92 0.92 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to dummy_test/model.dat at epoch: 0 and macro avg/f1-score: 0.5424475654733077 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.80 0.73 793\n", + " 1 0.61 0.17 0.27 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.75 0.64 0.65 7188\n", + "weighted avg 0.93 0.93 0.93 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.76 0.72 209\n", + " 1 0.36 0.36 0.36 11\n", + " 2 0.96 0.95 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.67 0.69 0.68 1797\n", + "weighted avg 0.93 0.93 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to dummy_test/model.dat at epoch: 1 and macro avg/f1-score: 0.6809929509005487 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.73 0.88 0.79 793\n", + " 1 0.47 0.44 0.46 64\n", + " 2 0.98 0.95 0.97 6331\n", + "\n", + " accuracy 0.94 7188\n", + " macro avg 0.73 0.76 0.74 7188\n", + "weighted avg 0.95 0.94 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.81 0.76 209\n", + " 1 0.31 0.45 0.37 11\n", + " 2 0.97 0.95 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.67 0.74 0.70 1797\n", + "weighted avg 0.94 0.93 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to dummy_test/model.dat at epoch: 2 and macro avg/f1-score: 0.6964827596275013 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.80 0.91 0.85 793\n", + " 1 0.62 0.73 0.67 64\n", + " 2 0.99 0.97 0.98 6331\n", + "\n", + " accuracy 0.96 7188\n", + " macro avg 0.80 0.87 0.83 7188\n", + "weighted avg 0.96 0.96 0.96 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.75 0.78 0.76 209\n", + " 1 0.31 0.36 0.33 11\n", + " 2 0.97 0.96 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.68 0.70 0.69 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.84 0.92 0.88 793\n", + " 1 0.73 0.91 0.81 64\n", + " 2 0.99 0.98 0.98 6331\n", + "\n", + " accuracy 0.97 7188\n", + " macro avg 0.86 0.94 0.89 7188\n", + "weighted avg 0.97 0.97 0.97 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.75 0.78 0.77 209\n", + " 1 0.27 0.36 0.31 11\n", + " 2 0.97 0.96 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.66 0.70 0.68 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.88 0.94 0.91 793\n", + " 1 0.81 0.92 0.86 64\n", + " 2 0.99 0.98 0.99 6331\n", + "\n", + " accuracy 0.98 7188\n", + " macro avg 0.89 0.95 0.92 7188\n", + "weighted avg 0.98 0.98 0.98 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.76 0.76 0.76 209\n", + " 1 0.29 0.36 0.32 11\n", + " 2 0.97 0.96 0.96 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.67 0.69 0.68 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.89 0.94 0.91 793\n", + " 1 0.85 0.95 0.90 64\n", + " 2 0.99 0.98 0.99 6331\n", + "\n", + " accuracy 0.98 7188\n", + " macro avg 0.91 0.96 0.93 7188\n", + "weighted avg 0.98 0.98 0.98 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.80 0.76 0.78 209\n", + " 1 0.44 0.36 0.40 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.74 0.70 0.72 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to dummy_test/model.dat at epoch: 6 and macro avg/f1-score: 0.7163631972056036 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.90 0.94 0.92 793\n", + " 1 0.91 0.97 0.94 64\n", + " 2 0.99 0.99 0.99 6331\n", + "\n", + " accuracy 0.98 7188\n", + " macro avg 0.94 0.97 0.95 7188\n", + "weighted avg 0.98 0.98 0.98 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.80 0.76 0.78 209\n", + " 1 0.44 0.36 0.40 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.74 0.70 0.72 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.91 0.96 0.94 793\n", + " 1 0.94 0.98 0.96 64\n", + " 2 0.99 0.99 0.99 6331\n", + "\n", + " accuracy 0.98 7188\n", + " macro avg 0.95 0.98 0.96 7188\n", + "weighted avg 0.99 0.98 0.98 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.78 0.78 209\n", + " 1 0.40 0.36 0.38 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.72 0.70 0.71 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.92 0.96 0.94 793\n", + " 1 0.93 0.98 0.95 64\n", + " 2 0.99 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.95 0.98 0.96 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.81 0.79 209\n", + " 1 0.38 0.45 0.42 11\n", + " 2 0.97 0.96 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.71 0.74 0.73 1797\n", + "weighted avg 0.95 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to dummy_test/model.dat at epoch: 9 and macro avg/f1-score: 0.7253847166674895 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.92 0.97 0.94 793\n", + " 1 0.96 1.00 0.98 64\n", + " 2 1.00 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.96 0.99 0.97 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.78 0.79 0.79 209\n", + " 1 0.33 0.45 0.38 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.69 0.74 0.71 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.92 0.97 0.94 793\n", + " 1 0.94 1.00 0.97 64\n", + " 2 1.00 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.95 0.99 0.97 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.77 0.78 209\n", + " 1 0.40 0.36 0.38 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.72 0.70 0.71 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.94 0.97 0.95 793\n", + " 1 0.96 1.00 0.98 64\n", + " 2 1.00 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.96 0.99 0.97 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.76 0.79 209\n", + " 1 0.36 0.36 0.36 11\n", + " 2 0.97 0.97 0.97 1577\n", + "\n", + " accuracy 0.95 1797\n", + " macro avg 0.71 0.70 0.71 1797\n", + "weighted avg 0.94 0.95 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.94 0.98 0.96 793\n", + " 1 0.96 1.00 0.98 64\n", + " 2 1.00 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.96 0.99 0.98 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.82 0.74 0.78 209\n", + " 1 0.40 0.36 0.38 11\n", + " 2 0.96 0.98 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.73 0.69 0.71 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.93 0.98 0.95 793\n", + " 1 0.96 1.00 0.98 64\n", + " 2 1.00 0.99 0.99 6331\n", + "\n", + " accuracy 0.99 7188\n", + " macro avg 0.96 0.99 0.98 7188\n", + "weighted avg 0.99 0.99 0.99 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.75 0.78 209\n", + " 1 0.40 0.36 0.38 11\n", + " 2 0.96 0.97 0.97 1577\n", + "\n", + " accuracy 0.94 1797\n", + " macro avg 0.72 0.69 0.71 1797\n", + "weighted avg 0.94 0.94 0.94 1797\n", + "\n" + ] + } + ], + "source": [ + "for meta_model in meta_model_names:\n", + " vocab_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-vocab.json')\n", + " merges_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,'bbpe-merges.txt')\n", + " tokenizer = TokenizerWrapperBPE(ByteLevelBPETokenizer(vocab=vocab_file,\n", + " merges=merges_file,\n", + " lowercase=True))\n", + " # load and sort out the config\n", + " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", + " with open(config_file, 'r') as jfile:\n", + " config_dict = json.load(jfile)\n", + " config = ConfigMetaCAT()\n", + " for key, value in config_dict.items():\n", + " setattr(config, key, value['py/state']['__dict__'])\n", + " \n", + " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", + " #Ideally this should replace the meta_models inside the modelpack\n", + " \n", + " #Below are the config values used for Experiencer classification task\n", + " \n", + " #Class weights--------------------------------------------------------------\n", + " #adjusting class weights to give more importance to minority classes\n", + " # To use class weights, we have 2 options:\n", + "\n", + " #1st option:\n", + " #to calculate class weights based on class distribution\n", + " #NOTE: this will only be applicable if config.train.class_weights is empty\n", + " config.train['class_weights'] = []\n", + " config.train['compute_class_weights'] = True\n", + "\n", + " #2nd option\n", + " #using specified class weights\n", + " config.train['class_weights'] = [0.4,1.5,0.1]\n", + "\n", + " #we'll use the 2nd option in this example\n", + " #----------------------------------------------------------------------------\n", + " \n", + " #NOTE: when using class weights, it is recommended to define the category to index mapping to ensure the weights are assigned to the right class\n", + " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", + "\n", + " config.train['test_size'] = 0.2\n", + " config.train['nepochs'] = 15\n", + "\n", + " #since we have class imbalance, macro avg is better suited than weighted avg\n", + " config.train.metric['base'] = 'macro avg'\n", + "\n", + " # Initialise and train meta_model\n", + " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" + ] + }, + { + "cell_type": "markdown", + "id": "0010d730", + "metadata": {}, + "source": [ + "LSTM has high weighted F1-score (0.94), however due to the class imbalance, that can be misleading.
The recall values for the minority classes (Other and Family) are low, especially for Family - 0.36\n", + "
There is room for improvement in performance, especially for the minority classes" + ] + }, + { + "cell_type": "markdown", + "id": "91ff4e28", + "metadata": {}, + "source": [ + "# For BERT model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "521479e4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", + "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", + "WARNING:medcat.utils.meta_cat.models:\n", + "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", + "INFO:medcat.meta_cat:BERT model used for classification\n", + "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", + "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {1: 75, 0: 75, 2: 75}\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n", + "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 1078\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.12 0.25 0.16 793\n", + " 1 0.00 0.03 0.01 64\n", + " 2 0.88 0.68 0.77 6331\n", + "\n", + " accuracy 0.63 7188\n", + " macro avg 0.34 0.32 0.31 7188\n", + "weighted avg 0.79 0.63 0.70 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.00 0.00 0.00 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.88 1.00 0.93 1577\n", + "\n", + " accuracy 0.88 1797\n", + " macro avg 0.29 0.33 0.31 1797\n", + "weighted avg 0.77 0.88 0.82 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.31159849832049 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.14 0.03 0.05 793\n", + " 1 0.00 0.00 0.00 64\n", + " 2 0.88 0.98 0.93 6331\n", + "\n", + " accuracy 0.86 7188\n", + " macro avg 0.34 0.34 0.33 7188\n", + "weighted avg 0.79 0.86 0.82 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.00 0.00 0.00 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.88 1.00 0.93 1577\n", + "\n", + " accuracy 0.88 1797\n", + " macro avg 0.29 0.33 0.31 1797\n", + "weighted avg 0.77 0.88 0.82 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.26 0.06 0.10 793\n", + " 1 0.00 0.00 0.00 64\n", + " 2 0.88 0.98 0.93 6331\n", + "\n", + " accuracy 0.87 7188\n", + " macro avg 0.38 0.35 0.34 7188\n", + "weighted avg 0.81 0.87 0.83 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.89 0.08 0.15 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.89 1.00 0.94 1577\n", + "\n", + " accuracy 0.89 1797\n", + " macro avg 0.59 0.36 0.36 1797\n", + "weighted avg 0.88 0.89 0.84 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.36267332514096967 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.59 0.42 0.49 793\n", + " 1 0.00 0.00 0.00 64\n", + " 2 0.92 0.96 0.94 6331\n", + "\n", + " accuracy 0.90 7188\n", + " macro avg 0.50 0.46 0.48 7188\n", + "weighted avg 0.88 0.90 0.88 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.61 0.68 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.94 0.98 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.57 0.53 0.55 1797\n", + "weighted avg 0.92 0.93 0.92 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 3 and macro avg/f1-score: 0.5474452879879955 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.61 0.66 0.63 793\n", + " 1 0.25 0.02 0.03 64\n", + " 2 0.95 0.95 0.95 6331\n", + "\n", + " accuracy 0.91 7188\n", + " macro avg 0.60 0.54 0.54 7188\n", + "weighted avg 0.91 0.91 0.91 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.62 0.68 0.65 209\n", + " 1 0.00 0.00 0.00 11\n", + " 2 0.96 0.95 0.95 1577\n", + "\n", + " accuracy 0.91 1797\n", + " macro avg 0.52 0.54 0.53 1797\n", + "weighted avg 0.91 0.91 0.91 1797\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.62 0.70 0.66 793\n", + " 1 0.68 0.23 0.35 64\n", + " 2 0.96 0.95 0.96 6331\n", + "\n", + " accuracy 0.92 7188\n", + " macro avg 0.75 0.63 0.65 7188\n", + "weighted avg 0.92 0.92 0.92 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.62 0.71 0.67 209\n", + " 1 0.45 0.45 0.45 11\n", + " 2 0.96 0.94 0.95 1577\n", + "\n", + " accuracy 0.91 1797\n", + " macro avg 0.68 0.70 0.69 1797\n", + "weighted avg 0.92 0.91 0.91 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 5 and macro avg/f1-score: 0.6901427504420462 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.71 0.69 793\n", + " 1 0.60 0.56 0.58 64\n", + " 2 0.96 0.95 0.96 6331\n", + "\n", + " accuracy 0.92 7188\n", + " macro avg 0.74 0.74 0.74 7188\n", + "weighted avg 0.93 0.92 0.92 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.65 0.71 0.68 209\n", + " 1 0.39 0.64 0.48 11\n", + " 2 0.96 0.94 0.95 1577\n", + "\n", + " accuracy 0.91 1797\n", + " macro avg 0.66 0.76 0.70 1797\n", + "weighted avg 0.92 0.91 0.92 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 6 and macro avg/f1-score: 0.7039037826541276 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.72 0.70 793\n", + " 1 0.52 0.53 0.53 64\n", + " 2 0.96 0.96 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.72 0.74 0.73 7188\n", + "weighted avg 0.93 0.93 0.93 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.63 0.73 0.68 209\n", + " 1 0.41 0.64 0.50 11\n", + " 2 0.96 0.94 0.95 1577\n", + "\n", + " accuracy 0.91 1797\n", + " macro avg 0.67 0.77 0.71 1797\n", + "weighted avg 0.92 0.91 0.92 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 7 and macro avg/f1-score: 0.7085684968468765 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.75 0.71 793\n", + " 1 0.62 0.53 0.57 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.75 0.74 0.75 7188\n", + "weighted avg 0.93 0.93 0.93 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.72 0.69 209\n", + " 1 0.40 0.73 0.52 11\n", + " 2 0.96 0.95 0.95 1577\n", + "\n", + " accuracy 0.92 1797\n", + " macro avg 0.68 0.80 0.72 1797\n", + "weighted avg 0.92 0.92 0.92 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 8 and macro avg/f1-score: 0.7214487000648947 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.77 0.73 793\n", + " 1 0.58 0.66 0.62 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.75 0.79 0.77 7188\n", + "weighted avg 0.94 0.93 0.93 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.74 0.71 209\n", + " 1 0.44 0.73 0.55 11\n", + " 2 0.96 0.95 0.96 1577\n", + "\n", + " accuracy 0.92 1797\n", + " macro avg 0.70 0.81 0.74 1797\n", + "weighted avg 0.93 0.92 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 9 and macro avg/f1-score: 0.7411771369414074 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.79 0.74 793\n", + " 1 0.65 0.72 0.68 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.77 0.82 0.79 7188\n", + "weighted avg 0.94 0.93 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.74 0.73 209\n", + " 1 0.42 0.73 0.53 11\n", + " 2 0.96 0.96 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.70 0.81 0.74 1797\n", + "weighted avg 0.93 0.93 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.79 0.74 793\n", + " 1 0.63 0.72 0.67 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.77 0.82 0.79 7188\n", + "weighted avg 0.94 0.93 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.75 0.74 209\n", + " 1 0.40 0.73 0.52 11\n", + " 2 0.97 0.96 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.70 0.81 0.74 1797\n", + "weighted avg 0.93 0.93 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.81 0.74 793\n", + " 1 0.60 0.66 0.63 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.75 0.81 0.78 7188\n", + "weighted avg 0.94 0.93 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.75 0.74 209\n", + " 1 0.40 0.73 0.52 11\n", + " 2 0.97 0.96 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.70 0.81 0.74 1797\n", + "weighted avg 0.94 0.93 0.93 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.80 0.75 793\n", + " 1 0.60 0.78 0.68 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.76 0.84 0.79 7188\n", + "weighted avg 0.94 0.93 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.77 0.75 209\n", + " 1 0.40 0.73 0.52 11\n", + " 2 0.97 0.96 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.70 0.82 0.74 1797\n", + "weighted avg 0.94 0.93 0.94 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 13 and macro avg/f1-score: 0.7440346627353422 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.80 0.75 793\n", + " 1 0.56 0.67 0.61 64\n", + " 2 0.97 0.95 0.96 6331\n", + "\n", + " accuracy 0.93 7188\n", + " macro avg 0.75 0.81 0.77 7188\n", + "weighted avg 0.94 0.93 0.94 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.76 0.75 209\n", + " 1 0.40 0.73 0.52 11\n", + " 2 0.97 0.96 0.96 1577\n", + "\n", + " accuracy 0.93 1797\n", + " macro avg 0.70 0.82 0.74 1797\n", + "weighted avg 0.94 0.93 0.94 1797\n", + "\n" + ] + } + ], + "source": [ + "for meta_model in meta_model_names:\n", + " # load and sort out the config\n", + " config_file = os.path.join(base_dir_meta_models,\"meta_\"+meta_model,\"config.json\")\n", + " with open(config_file, 'r') as jfile:\n", + " config_dict = json.load(jfile)\n", + " config = ConfigMetaCAT()\n", + " for key, value in config_dict.items():\n", + " setattr(config, key, value['py/state']['__dict__'])\n", + "\n", + " # change model name if training BERT for the first time\n", + " config.model['model_name'] = 'bert'\n", + "\n", + " tokenizer = TokenizerWrapperBERT.load(os.path.join(base_dir_meta_models,\"meta_\"+meta_model), config.model['model_variant'])\n", + " \n", + " save_dir_path= \"test_meta_\"+meta_model # Where to save the meta_model and results. \n", + " #Ideally this should replace the meta_models inside the modelpack\n", + "\n", + " #Below are the config values used for Experiencer classification task\n", + "\n", + " config.model['nclasses'] = 3\n", + " config.general['category_name'] = 'Experiencer'\n", + "\n", + " config.train.lr = 5e-4\n", + " config.train['test_size'] = 0.2\n", + " config.train['nepochs'] = 15\n", + "\n", + " # you can also switch between freezing BERT layers or using LoRA during training\n", + " # to use LORA:\n", + " config.model['model_freeze_layers'] = False\n", + "\n", + " config.train.metric['base'] = 'macro avg'\n", + "\n", + " config.train['class_weights'] = [0.4,1.5,0.1]\n", + " config.general['category_value2id'] = {'Family':1, 'Other':0, 'Patient':2}\n", + "\n", + " save_dir_path= \"test_meta\" # Where to save the meta_model and results.\n", + " #Ideally this should replace the meta_models inside the modelpack\n", + "\n", + " # Initialise and train meta_model\n", + " mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + " results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + "\n", + " # Save results\n", + " json.dump(results, open(os.path.join(save_dir_path,'meta_results.json'), 'w'))" + ] + }, + { + "cell_type": "markdown", + "id": "8dc9496b", + "metadata": {}, + "source": [ + "BERT model shows improvement for the Family class compared to LSTM, however the recall values can still be improved.
To help tackle this, we'll use 2 phase learning for training." + ] + }, + { + "cell_type": "markdown", + "id": "ab23e424", + "metadata": {}, + "source": [ + "## If you dont have the model packs, and are training from scratch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16231060", + "metadata": {}, + "outputs": [], + "source": [ + "config = ConfigMetaCAT()\n", + "# make sure to change the following parameters:\n", + "# config.model['nclasses']\n", + "# config.general['category_name']\n", + "\n", + "# change model name if training BERT for the first time\n", + "config.model['model_name'] = 'bert'\n", + "\n", + "tokenizer = TokenizerWrapperBERT.load(\"\", config.model['model_variant'])\n", + "\n", + "save_dir_path= \"test_meta\" # Where to save the meta_model and results. \n", + "#Ideally this should replace the meta_models inside the modelpack\n", + "\n", + "# Initialise and train meta_model\n", + "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)\n", + "\n", + "# Save results\n", + "json.dump(results, open(os.path.join(save_dir_path,'meta_results.json'), 'w'))" + ] + }, + { + "cell_type": "markdown", + "id": "c3002ef0", + "metadata": {}, + "source": [ + "## If using 2 phase learning for training" + ] + }, + { + "cell_type": "markdown", + "id": "b83d9958", + "metadata": {}, + "source": [ + "### Phase 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9bcdf7be", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", + "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", + "WARNING:medcat.utils.meta_cat.models:\n", + "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", + "INFO:medcat.meta_cat:BERT model used for classification\n", + "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", + "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n", + "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 665\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.48 0.80 0.60 809\n", + " 1 0.03 0.03 0.03 61\n", + " 2 0.44 0.13 0.20 793\n", + "\n", + " accuracy 0.45 1663\n", + " macro avg 0.32 0.32 0.27 1663\n", + "weighted avg 0.44 0.45 0.39 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.47 1.00 0.63 193\n", + " 1 0.00 0.00 0.00 14\n", + " 2 1.00 0.00 0.01 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.49 0.33 0.21 416\n", + "weighted avg 0.72 0.47 0.30 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 0 and macro avg/f1-score: 0.21479741019214701 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.48 0.85 0.62 809\n", + " 1 0.06 0.08 0.07 61\n", + " 2 0.44 0.09 0.14 793\n", + "\n", + " accuracy 0.46 1663\n", + " macro avg 0.33 0.34 0.28 1663\n", + "weighted avg 0.45 0.46 0.37 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.47 1.00 0.63 193\n", + " 1 1.00 0.07 0.13 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.49 0.36 0.26 416\n", + "weighted avg 0.25 0.47 0.30 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 1 and macro avg/f1-score: 0.2560672514619883 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.50 0.89 0.64 809\n", + " 1 0.14 0.36 0.20 61\n", + " 2 0.58 0.05 0.09 793\n", + "\n", + " accuracy 0.47 1663\n", + " macro avg 0.41 0.43 0.31 1663\n", + "weighted avg 0.53 0.47 0.36 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.47 0.97 0.63 193\n", + " 1 0.31 0.36 0.33 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.46 416\n", + " macro avg 0.26 0.44 0.32 416\n", + "weighted avg 0.23 0.46 0.31 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 2 and macro avg/f1-score: 0.322465804759228 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.53 0.90 0.67 809\n", + " 1 0.15 0.66 0.24 61\n", + " 2 0.41 0.01 0.02 793\n", + "\n", + " accuracy 0.47 1663\n", + " macro avg 0.36 0.52 0.31 1663\n", + "weighted avg 0.46 0.47 0.34 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.58 0.90 0.71 193\n", + " 1 0.09 0.79 0.17 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.44 416\n", + " macro avg 0.23 0.56 0.29 416\n", + "weighted avg 0.27 0.44 0.33 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.56 0.92 0.70 809\n", + " 1 0.15 0.80 0.26 61\n", + " 2 0.75 0.01 0.01 793\n", + "\n", + " accuracy 0.48 1663\n", + " macro avg 0.49 0.58 0.32 1663\n", + "weighted avg 0.64 0.48 0.36 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.60 0.96 0.74 193\n", + " 1 0.08 0.64 0.15 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.23 0.53 0.30 416\n", + "weighted avg 0.28 0.47 0.35 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.58 0.96 0.73 809\n", + " 1 0.15 0.82 0.26 61\n", + " 2 1.00 0.00 0.01 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.58 0.59 0.33 1663\n", + "weighted avg 0.77 0.50 0.36 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.65 0.94 0.77 193\n", + " 1 0.07 0.71 0.13 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.46 416\n", + " macro avg 0.24 0.55 0.30 416\n", + "weighted avg 0.31 0.46 0.36 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.63 0.94 0.75 809\n", + " 1 0.13 0.90 0.22 61\n", + " 2 1.00 0.01 0.01 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.58 0.62 0.33 1663\n", + "weighted avg 0.79 0.50 0.38 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.65 0.95 0.77 193\n", + " 1 0.09 0.86 0.16 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.25 0.60 0.31 416\n", + "weighted avg 0.30 0.47 0.36 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.64 0.95 0.76 809\n", + " 1 0.12 0.93 0.22 61\n", + " 2 0.75 0.00 0.01 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.50 0.63 0.33 1663\n", + "weighted avg 0.67 0.50 0.38 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.65 0.94 0.77 193\n", + " 1 0.09 0.86 0.16 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.46 416\n", + " macro avg 0.25 0.60 0.31 416\n", + "weighted avg 0.31 0.46 0.36 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.95 0.78 809\n", + " 1 0.12 0.95 0.21 61\n", + " 2 1.00 0.01 0.02 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.59 0.64 0.34 1663\n", + "weighted avg 0.81 0.50 0.40 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.95 0.78 193\n", + " 1 0.09 0.86 0.16 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.25 0.60 0.31 416\n", + "weighted avg 0.31 0.47 0.37 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.95 0.78 809\n", + " 1 0.12 0.95 0.21 61\n", + " 2 1.00 0.00 0.01 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.59 0.64 0.33 1663\n", + "weighted avg 0.80 0.50 0.39 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.94 0.78 193\n", + " 1 0.08 0.86 0.15 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.47 416\n", + " macro avg 0.25 0.60 0.31 416\n", + "weighted avg 0.31 0.47 0.37 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.95 0.79 809\n", + " 1 0.12 0.97 0.21 61\n", + " 2 1.00 0.01 0.02 793\n", + "\n", + " accuracy 0.51 1663\n", + " macro avg 0.60 0.64 0.34 1663\n", + "weighted avg 0.81 0.51 0.40 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 10 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.97 0.80 193\n", + " 1 0.09 0.93 0.17 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.26 0.63 0.32 416\n", + "weighted avg 0.32 0.48 0.38 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 10 and macro avg/f1-score: 0.32250773257197235 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.95 0.80 809\n", + " 1 0.11 0.97 0.20 61\n", + " 2 0.77 0.01 0.02 793\n", + "\n", + " accuracy 0.50 1663\n", + " macro avg 0.52 0.64 0.34 1663\n", + "weighted avg 0.71 0.50 0.41 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 11 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.97 0.79 193\n", + " 1 0.09 0.93 0.17 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.26 0.63 0.32 416\n", + "weighted avg 0.31 0.48 0.37 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.96 0.80 809\n", + " 1 0.12 1.00 0.22 61\n", + " 2 1.00 0.02 0.03 793\n", + "\n", + " accuracy 0.51 1663\n", + " macro avg 0.60 0.66 0.35 1663\n", + "weighted avg 0.81 0.51 0.41 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 12 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.97 0.80 193\n", + " 1 0.09 0.93 0.17 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.26 0.63 0.32 416\n", + "weighted avg 0.32 0.48 0.38 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 12 and macro avg/f1-score: 0.3237202463008914 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.96 0.80 809\n", + " 1 0.12 0.98 0.22 61\n", + " 2 0.96 0.03 0.06 793\n", + "\n", + " accuracy 0.52 1663\n", + " macro avg 0.59 0.66 0.36 1663\n", + "weighted avg 0.79 0.52 0.42 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 13 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.98 0.79 193\n", + " 1 0.09 0.79 0.15 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.25 0.59 0.31 416\n", + "weighted avg 0.31 0.48 0.37 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.97 0.81 809\n", + " 1 0.12 1.00 0.21 61\n", + " 2 0.88 0.03 0.06 793\n", + "\n", + " accuracy 0.52 1663\n", + " macro avg 0.57 0.67 0.36 1663\n", + "weighted avg 0.77 0.52 0.43 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 14 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.98 0.81 193\n", + " 1 0.08 0.86 0.15 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.26 0.61 0.32 416\n", + "weighted avg 0.32 0.48 0.38 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.97 0.82 809\n", + " 1 0.12 1.00 0.21 61\n", + " 2 0.95 0.05 0.10 793\n", + "\n", + " accuracy 0.53 1663\n", + " macro avg 0.59 0.67 0.38 1663\n", + "weighted avg 0.80 0.53 0.45 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 15 ************************************************** Test\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.98 0.81 193\n", + " 1 0.09 0.86 0.16 14\n", + " 2 0.00 0.00 0.00 209\n", + "\n", + " accuracy 0.48 416\n", + " macro avg 0.26 0.61 0.32 416\n", + "weighted avg 0.32 0.48 0.38 416\n", + "\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.97 0.81 809\n", + " 1 0.12 1.00 0.22 61\n", + " 2 0.94 0.04 0.07 793\n", + "\n", + " accuracy 0.53 1663\n", + " macro avg 0.58 0.67 0.37 1663\n", + "weighted avg 0.79 0.53 0.44 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 16 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.98 0.80 193\n", + " 1 0.09 0.86 0.16 14\n", + " 2 1.00 0.00 0.01 209\n", + "\n", + " accuracy 0.49 416\n", + " macro avg 0.59 0.61 0.32 416\n", + "weighted avg 0.82 0.49 0.38 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.97 0.81 809\n", + " 1 0.12 1.00 0.22 61\n", + " 2 0.94 0.06 0.12 793\n", + "\n", + " accuracy 0.54 1663\n", + " macro avg 0.59 0.68 0.39 1663\n", + "weighted avg 0.80 0.54 0.46 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 17 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.66 0.98 0.79 193\n", + " 1 0.09 0.86 0.17 14\n", + " 2 1.00 0.02 0.04 209\n", + "\n", + " accuracy 0.49 416\n", + " macro avg 0.59 0.62 0.33 416\n", + "weighted avg 0.81 0.49 0.39 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 17 and macro avg/f1-score: 0.3328554768276514 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.98 0.81 809\n", + " 1 0.14 1.00 0.24 61\n", + " 2 0.95 0.08 0.15 793\n", + "\n", + " accuracy 0.55 1663\n", + " macro avg 0.59 0.69 0.40 1663\n", + "weighted avg 0.80 0.55 0.47 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 18 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.67 0.98 0.80 193\n", + " 1 0.09 0.86 0.17 14\n", + " 2 1.00 0.04 0.07 209\n", + "\n", + " accuracy 0.50 416\n", + " macro avg 0.59 0.62 0.35 416\n", + "weighted avg 0.82 0.50 0.41 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 18 and macro avg/f1-score: 0.3471379464272786 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.98 0.82 809\n", + " 1 0.13 1.00 0.24 61\n", + " 2 0.97 0.09 0.17 793\n", + "\n", + " accuracy 0.56 1663\n", + " macro avg 0.60 0.69 0.41 1663\n", + "weighted avg 0.81 0.56 0.49 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 19 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.98 0.80 193\n", + " 1 0.10 0.86 0.17 14\n", + " 2 1.00 0.07 0.13 209\n", + "\n", + " accuracy 0.52 416\n", + " macro avg 0.59 0.63 0.37 416\n", + "weighted avg 0.82 0.52 0.44 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 19 and macro avg/f1-score: 0.3674925759230418 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.98 0.83 809\n", + " 1 0.13 1.00 0.23 61\n", + " 2 0.94 0.10 0.18 793\n", + "\n", + " accuracy 0.56 1663\n", + " macro avg 0.59 0.69 0.41 1663\n", + "weighted avg 0.80 0.56 0.49 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 20 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.98 0.81 193\n", + " 1 0.10 0.86 0.18 14\n", + " 2 1.00 0.10 0.17 209\n", + "\n", + " accuracy 0.53 416\n", + " macro avg 0.60 0.64 0.39 416\n", + "weighted avg 0.83 0.53 0.47 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 20 and macro avg/f1-score: 0.38685497295333987 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.98 0.81 809\n", + " 1 0.14 1.00 0.25 61\n", + " 2 0.96 0.12 0.22 793\n", + "\n", + " accuracy 0.57 1663\n", + " macro avg 0.60 0.70 0.43 1663\n", + "weighted avg 0.80 0.57 0.51 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 21 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.68 0.98 0.81 193\n", + " 1 0.11 0.86 0.19 14\n", + " 2 0.96 0.11 0.21 209\n", + "\n", + " accuracy 0.54 416\n", + " macro avg 0.58 0.65 0.40 416\n", + "weighted avg 0.80 0.54 0.48 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 21 and macro avg/f1-score: 0.4002297137890358 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.70 0.99 0.82 809\n", + " 1 0.14 1.00 0.25 61\n", + " 2 0.98 0.12 0.22 793\n", + "\n", + " accuracy 0.57 1663\n", + " macro avg 0.61 0.70 0.43 1663\n", + "weighted avg 0.81 0.57 0.51 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 22 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.98 0.81 193\n", + " 1 0.11 0.86 0.20 14\n", + " 2 0.94 0.15 0.26 209\n", + "\n", + " accuracy 0.56 416\n", + " macro avg 0.58 0.66 0.42 416\n", + "weighted avg 0.80 0.56 0.51 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 22 and macro avg/f1-score: 0.4231725462046246 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.99 0.83 809\n", + " 1 0.15 1.00 0.26 61\n", + " 2 0.99 0.17 0.29 793\n", + "\n", + " accuracy 0.60 1663\n", + " macro avg 0.62 0.72 0.46 1663\n", + "weighted avg 0.82 0.60 0.55 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 23 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.69 0.99 0.81 193\n", + " 1 0.12 0.86 0.21 14\n", + " 2 1.00 0.19 0.31 209\n", + "\n", + " accuracy 0.58 416\n", + " macro avg 0.60 0.68 0.45 416\n", + "weighted avg 0.83 0.58 0.54 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 23 and macro avg/f1-score: 0.4459035717026967 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.99 0.83 809\n", + " 1 0.16 1.00 0.28 61\n", + " 2 0.99 0.21 0.35 793\n", + "\n", + " accuracy 0.62 1663\n", + " macro avg 0.62 0.73 0.49 1663\n", + "weighted avg 0.83 0.62 0.58 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 24 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.98 0.83 193\n", + " 1 0.12 0.86 0.21 14\n", + " 2 0.98 0.25 0.40 209\n", + "\n", + " accuracy 0.61 416\n", + " macro avg 0.61 0.70 0.48 416\n", + "weighted avg 0.83 0.61 0.60 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 24 and macro avg/f1-score: 0.482945795402635 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.71 0.99 0.83 809\n", + " 1 0.15 1.00 0.27 61\n", + " 2 0.98 0.18 0.31 793\n", + "\n", + " accuracy 0.60 1663\n", + " macro avg 0.62 0.72 0.47 1663\n", + "weighted avg 0.82 0.60 0.56 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 25 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.98 0.83 193\n", + " 1 0.13 0.86 0.22 14\n", + " 2 0.98 0.27 0.43 209\n", + "\n", + " accuracy 0.62 416\n", + " macro avg 0.61 0.70 0.49 416\n", + "weighted avg 0.83 0.62 0.61 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 25 and macro avg/f1-score: 0.493566120394729 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.72 0.99 0.83 809\n", + " 1 0.17 1.00 0.29 61\n", + " 2 0.99 0.24 0.39 793\n", + "\n", + " accuracy 0.63 1663\n", + " macro avg 0.63 0.74 0.50 1663\n", + "weighted avg 0.83 0.63 0.60 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 26 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.98 0.84 193\n", + " 1 0.14 0.86 0.24 14\n", + " 2 0.97 0.34 0.50 209\n", + "\n", + " accuracy 0.65 416\n", + " macro avg 0.62 0.73 0.53 416\n", + "weighted avg 0.84 0.65 0.65 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 26 and macro avg/f1-score: 0.52768022857349 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.98 0.85 809\n", + " 1 0.17 1.00 0.29 61\n", + " 2 0.98 0.28 0.43 793\n", + "\n", + " accuracy 0.65 1663\n", + " macro avg 0.63 0.75 0.52 1663\n", + "weighted avg 0.84 0.65 0.63 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 27 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.98 0.84 193\n", + " 1 0.15 0.86 0.25 14\n", + " 2 0.97 0.37 0.53 209\n", + "\n", + " accuracy 0.67 416\n", + " macro avg 0.62 0.73 0.54 416\n", + "weighted avg 0.84 0.67 0.67 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 27 and macro avg/f1-score: 0.5428240740740741 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.99 0.85 809\n", + " 1 0.18 1.00 0.31 61\n", + " 2 0.98 0.31 0.48 793\n", + "\n", + " accuracy 0.67 1663\n", + " macro avg 0.64 0.77 0.54 1663\n", + "weighted avg 0.84 0.67 0.65 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 28 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.76 0.98 0.86 193\n", + " 1 0.15 0.86 0.26 14\n", + " 2 0.98 0.41 0.58 209\n", + "\n", + " accuracy 0.69 416\n", + " macro avg 0.63 0.75 0.56 416\n", + "weighted avg 0.85 0.69 0.70 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 28 and macro avg/f1-score: 0.5638621950678688 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.74 0.99 0.85 809\n", + " 1 0.19 1.00 0.32 61\n", + " 2 0.99 0.33 0.50 793\n", + "\n", + " accuracy 0.68 1663\n", + " macro avg 0.64 0.77 0.55 1663\n", + "weighted avg 0.84 0.68 0.66 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 29 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.76 0.98 0.85 193\n", + " 1 0.16 0.86 0.27 14\n", + " 2 0.98 0.43 0.59 209\n", + "\n", + " accuracy 0.70 416\n", + " macro avg 0.63 0.75 0.57 416\n", + "weighted avg 0.85 0.70 0.70 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 29 and macro avg/f1-score: 0.5720897974597215 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.76 0.99 0.86 809\n", + " 1 0.18 1.00 0.30 61\n", + " 2 0.99 0.34 0.51 793\n", + "\n", + " accuracy 0.68 1663\n", + " macro avg 0.64 0.78 0.56 1663\n", + "weighted avg 0.85 0.68 0.67 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 30 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.98 0.86 193\n", + " 1 0.17 0.86 0.29 14\n", + " 2 0.98 0.48 0.65 209\n", + "\n", + " accuracy 0.73 416\n", + " macro avg 0.64 0.77 0.60 416\n", + "weighted avg 0.86 0.73 0.74 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 30 and macro avg/f1-score: 0.6005270274314415 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.99 0.87 809\n", + " 1 0.21 1.00 0.35 61\n", + " 2 0.99 0.43 0.60 793\n", + "\n", + " accuracy 0.72 1663\n", + " macro avg 0.66 0.81 0.61 1663\n", + "weighted avg 0.86 0.72 0.72 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 31 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.98 0.86 193\n", + " 1 0.21 0.86 0.33 14\n", + " 2 0.98 0.54 0.69 209\n", + "\n", + " accuracy 0.75 416\n", + " macro avg 0.65 0.79 0.63 416\n", + "weighted avg 0.86 0.75 0.76 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 31 and macro avg/f1-score: 0.6306067812327066 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.98 0.87 809\n", + " 1 0.23 1.00 0.37 61\n", + " 2 0.98 0.46 0.63 793\n", + "\n", + " accuracy 0.73 1663\n", + " macro avg 0.66 0.81 0.62 1663\n", + "weighted avg 0.85 0.73 0.73 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 32 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.98 0.87 193\n", + " 1 0.21 0.86 0.34 14\n", + " 2 0.98 0.56 0.71 209\n", + "\n", + " accuracy 0.76 416\n", + " macro avg 0.66 0.80 0.64 416\n", + "weighted avg 0.87 0.76 0.77 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 32 and macro avg/f1-score: 0.6414740059803422 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.77 0.99 0.87 809\n", + " 1 0.23 1.00 0.38 61\n", + " 2 0.99 0.46 0.63 793\n", + "\n", + " accuracy 0.74 1663\n", + " macro avg 0.67 0.82 0.62 1663\n", + "weighted avg 0.86 0.74 0.74 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 33 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.98 0.87 193\n", + " 1 0.22 0.86 0.35 14\n", + " 2 0.98 0.57 0.72 209\n", + "\n", + " accuracy 0.77 416\n", + " macro avg 0.66 0.80 0.65 416\n", + "weighted avg 0.87 0.77 0.78 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 33 and macro avg/f1-score: 0.6473391409830812 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.76 0.98 0.86 809\n", + " 1 0.23 1.00 0.38 61\n", + " 2 0.99 0.45 0.62 793\n", + "\n", + " accuracy 0.73 1663\n", + " macro avg 0.66 0.81 0.62 1663\n", + "weighted avg 0.85 0.73 0.73 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 34 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.98 0.88 193\n", + " 1 0.27 0.86 0.41 14\n", + " 2 0.99 0.63 0.77 209\n", + "\n", + " accuracy 0.80 416\n", + " macro avg 0.68 0.82 0.69 416\n", + "weighted avg 0.87 0.80 0.81 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 34 and macro avg/f1-score: 0.6868341887176047 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.78 0.99 0.87 809\n", + " 1 0.26 1.00 0.41 61\n", + " 2 0.99 0.51 0.67 793\n", + "\n", + " accuracy 0.76 1663\n", + " macro avg 0.68 0.83 0.65 1663\n", + "weighted avg 0.86 0.76 0.76 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 35 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.97 0.89 193\n", + " 1 0.23 0.86 0.36 14\n", + " 2 0.98 0.62 0.76 209\n", + "\n", + " accuracy 0.79 416\n", + " macro avg 0.67 0.82 0.67 416\n", + "weighted avg 0.88 0.79 0.80 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.79 0.99 0.88 809\n", + " 1 0.24 1.00 0.39 61\n", + " 2 0.99 0.50 0.66 793\n", + "\n", + " accuracy 0.75 1663\n", + " macro avg 0.67 0.83 0.64 1663\n", + "weighted avg 0.87 0.75 0.76 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 36 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.97 0.89 193\n", + " 1 0.27 0.86 0.41 14\n", + " 2 0.98 0.66 0.79 209\n", + "\n", + " accuracy 0.81 416\n", + " macro avg 0.69 0.83 0.69 416\n", + "weighted avg 0.88 0.81 0.82 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 36 and macro avg/f1-score: 0.6928908001266963 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.80 0.98 0.88 809\n", + " 1 0.26 1.00 0.41 61\n", + " 2 0.98 0.53 0.69 793\n", + "\n", + " accuracy 0.77 1663\n", + " macro avg 0.68 0.84 0.66 1663\n", + "weighted avg 0.87 0.77 0.77 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 37 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.97 0.89 193\n", + " 1 0.27 0.86 0.41 14\n", + " 2 0.98 0.66 0.79 209\n", + "\n", + " accuracy 0.81 416\n", + " macro avg 0.69 0.83 0.70 416\n", + "weighted avg 0.88 0.81 0.82 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:\n", + "##### Model saved to test_meta/model.dat at epoch: 37 and macro avg/f1-score: 0.6963856616166311 #####\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.78 0.98 0.87 809\n", + " 1 0.25 1.00 0.41 61\n", + " 2 0.98 0.50 0.66 793\n", + "\n", + " accuracy 0.75 1663\n", + " macro avg 0.67 0.83 0.65 1663\n", + "weighted avg 0.86 0.75 0.75 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 38 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.97 0.89 193\n", + " 1 0.27 0.86 0.41 14\n", + " 2 0.98 0.66 0.79 209\n", + "\n", + " accuracy 0.81 416\n", + " macro avg 0.69 0.83 0.70 416\n", + "weighted avg 0.88 0.81 0.82 416\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.80 0.98 0.88 809\n", + " 1 0.26 1.00 0.41 61\n", + " 2 0.98 0.53 0.69 793\n", + "\n", + " accuracy 0.77 1663\n", + " macro avg 0.68 0.84 0.66 1663\n", + "weighted avg 0.86 0.77 0.77 1663\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 39 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.81 0.97 0.89 193\n", + " 1 0.27 0.86 0.41 14\n", + " 2 0.98 0.66 0.79 209\n", + "\n", + " accuracy 0.81 416\n", + " macro avg 0.69 0.83 0.70 416\n", + "weighted avg 0.88 0.81 0.82 416\n", + "\n" + ] + } + ], + "source": [ + "######################################################################################################\n", + "# 2 phase learning (used for imbalanced datasets) - trains the models twice: \n", + "# phase 1: trains for minority class(es) by undersampling data\n", + "# phase 2: trains for all classes\n", + "# parameter values: \n", + "# 1: Phase 1 - Train model on undersampled data\n", + "# 2: Phase 2 - Continue training on full data\n", + "# 0: None\n", + "#\n", + "# Paper reference - https://ieeexplore.ieee.org/document/7533053\n", + "# NOTE: Make sure to use class weights in favour of minority classes with 2 phase learning\n", + "#####################################################################################################\n", + "\n", + "# Follow same steps till defining save_dir_path\n", + "\n", + "#change phase number to 1\n", + "config.model.phase_number = 1\n", + "\n", + "# specify the class that will define the desired sample size for the undersampling process\n", + "# if this is left empty, the class with the lowest samples will be chosen\n", + "# example\n", + "config.model['category_undersample'] = 'Other'\n", + "\n", + "#Below are the config values used for Experiencer classification task\n", + "\n", + "config.model['nclasses'] = 3\n", + "config.general['category_name'] = 'Experiencer'\n", + "\n", + "config.model['category_undersample'] = 'Other'\n", + "\n", + "config.train.lr = 5e-4\n", + "config.train['test_size'] = 0.2\n", + "config.train['nepochs'] = 20\n", + "\n", + "config.train.metric['base'] = 'macro avg'\n", + "\n", + "config.train['class_weights'] = [0.4,1.5,0.05]\n", + "config.general['category_value2id'] = {'Other':0, 'Family':1, 'Patient':2}\n", + "\n", + "config.model['model_freeze_layers'] = False\n", + "\n", + "# Initialise and train meta_model \n", + "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" + ] + }, + { + "cell_type": "markdown", + "id": "e229ef0a", + "metadata": {}, + "source": [ + "## Phase 2" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "63c01002", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Could not load tokenizer from path due to error: bert-tokenizer is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\n", + "If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `huggingface-cli login` or by passing `token=`. Loading from library for model variant: bert-base-uncased\n", + "WARNING:medcat.utils.meta_cat.models:\n", + "Input size for bert-base-uncased model should be 768, provided input size is 300 Input size changed to 768\n", + "INFO:medcat.meta_cat:BERT model used for classification\n", + "Token indices sequence length is longer than the specified maximum sequence length for this model (554 > 512). Running this sequence through the model will result in indexing errors\n", + "INFO:medcat.utils.meta_cat.data_utils:Updated label_data: {0: 1002, 1: 75, 2: 1002}\n", + "INFO:medcat.meta_cat:Model state loaded from dict for 2 phase learning\n", + "/home/sagarwal/.local/lib/python3.8/site-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n", + "INFO:medcat.utils.meta_cat.ml_utils:Total steps for optimizer: 718\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.32 0.94 0.48 793\n", + " 1 0.07 0.97 0.13 64\n", + " 2 0.99 0.62 0.76 6331\n", + "\n", + " accuracy 0.66 7188\n", + " macro avg 0.46 0.84 0.45 7188\n", + "weighted avg 0.91 0.66 0.72 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 0 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.40 0.95 0.56 209\n", + " 1 0.10 1.00 0.18 11\n", + " 2 0.99 0.75 0.85 1577\n", + "\n", + " accuracy 0.77 1797\n", + " macro avg 0.50 0.90 0.53 1797\n", + "weighted avg 0.92 0.77 0.81 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.42 0.92 0.58 793\n", + " 1 0.15 0.97 0.26 64\n", + " 2 0.99 0.79 0.88 6331\n", + "\n", + " accuracy 0.80 7188\n", + " macro avg 0.52 0.89 0.57 7188\n", + "weighted avg 0.92 0.80 0.84 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 1 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.49 0.92 0.64 209\n", + " 1 0.12 1.00 0.21 11\n", + " 2 0.99 0.82 0.90 1577\n", + "\n", + " accuracy 0.83 1797\n", + " macro avg 0.53 0.91 0.58 1797\n", + "weighted avg 0.92 0.83 0.86 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.48 0.92 0.63 793\n", + " 1 0.21 0.97 0.35 64\n", + " 2 0.99 0.84 0.91 6331\n", + "\n", + " accuracy 0.85 7188\n", + " macro avg 0.56 0.91 0.63 7188\n", + "weighted avg 0.93 0.85 0.87 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 2 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.49 0.94 0.64 209\n", + " 1 0.13 1.00 0.23 11\n", + " 2 0.99 0.82 0.90 1577\n", + "\n", + " accuracy 0.84 1797\n", + " macro avg 0.54 0.92 0.59 1797\n", + "weighted avg 0.93 0.84 0.87 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.53 0.92 0.67 793\n", + " 1 0.25 0.95 0.40 64\n", + " 2 0.99 0.87 0.92 6331\n", + "\n", + " accuracy 0.87 7188\n", + " macro avg 0.59 0.91 0.66 7188\n", + "weighted avg 0.93 0.87 0.89 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 3 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.55 0.92 0.69 209\n", + " 1 0.14 1.00 0.25 11\n", + " 2 0.99 0.86 0.92 1577\n", + "\n", + " accuracy 0.87 1797\n", + " macro avg 0.56 0.93 0.62 1797\n", + "weighted avg 0.93 0.87 0.89 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.53 0.93 0.68 793\n", + " 1 0.31 0.95 0.47 64\n", + " 2 0.99 0.88 0.93 6331\n", + "\n", + " accuracy 0.88 7188\n", + " macro avg 0.61 0.92 0.69 7188\n", + "weighted avg 0.93 0.88 0.90 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 4 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.56 0.92 0.70 209\n", + " 1 0.15 1.00 0.27 11\n", + " 2 0.99 0.87 0.92 1577\n", + "\n", + " accuracy 0.87 1797\n", + " macro avg 0.57 0.93 0.63 1797\n", + "weighted avg 0.93 0.87 0.89 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.56 0.92 0.70 793\n", + " 1 0.32 0.92 0.47 64\n", + " 2 0.99 0.89 0.94 6331\n", + "\n", + " accuracy 0.89 7188\n", + " macro avg 0.62 0.91 0.70 7188\n", + "weighted avg 0.94 0.89 0.91 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 5 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.57 0.92 0.71 209\n", + " 1 0.17 1.00 0.29 11\n", + " 2 0.99 0.87 0.93 1577\n", + "\n", + " accuracy 0.88 1797\n", + " macro avg 0.58 0.93 0.64 1797\n", + "weighted avg 0.93 0.88 0.90 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.58 0.92 0.71 793\n", + " 1 0.34 0.95 0.50 64\n", + " 2 0.99 0.90 0.94 6331\n", + "\n", + " accuracy 0.90 7188\n", + " macro avg 0.63 0.92 0.72 7188\n", + "weighted avg 0.94 0.90 0.91 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 6 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.58 0.90 0.71 209\n", + " 1 0.16 1.00 0.28 11\n", + " 2 0.99 0.88 0.93 1577\n", + "\n", + " accuracy 0.88 1797\n", + " macro avg 0.58 0.93 0.64 1797\n", + "weighted avg 0.93 0.88 0.90 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.59 0.93 0.72 793\n", + " 1 0.32 0.97 0.48 64\n", + " 2 0.99 0.90 0.94 6331\n", + "\n", + " accuracy 0.90 7188\n", + " macro avg 0.63 0.93 0.71 7188\n", + "weighted avg 0.94 0.90 0.91 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 7 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.60 0.89 0.72 209\n", + " 1 0.19 1.00 0.31 11\n", + " 2 0.98 0.89 0.94 1577\n", + "\n", + " accuracy 0.89 1797\n", + " macro avg 0.59 0.93 0.66 1797\n", + "weighted avg 0.94 0.89 0.91 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.59 0.92 0.72 793\n", + " 1 0.35 0.97 0.51 64\n", + " 2 0.99 0.90 0.94 6331\n", + "\n", + " accuracy 0.91 7188\n", + " macro avg 0.64 0.93 0.73 7188\n", + "weighted avg 0.94 0.91 0.92 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 8 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.62 0.89 0.73 209\n", + " 1 0.20 1.00 0.33 11\n", + " 2 0.98 0.90 0.94 1577\n", + "\n", + " accuracy 0.90 1797\n", + " macro avg 0.60 0.93 0.67 1797\n", + "weighted avg 0.94 0.90 0.91 1797\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Train\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.62 0.92 0.74 793\n", + " 1 0.36 0.95 0.52 64\n", + " 2 0.99 0.91 0.95 6331\n", + "\n", + " accuracy 0.91 7188\n", + " macro avg 0.66 0.93 0.74 7188\n", + "weighted avg 0.94 0.91 0.92 7188\n", + "\n", + "INFO:medcat.utils.meta_cat.ml_utils:Epoch: 9 ************************************************** Test\n", + "INFO:medcat.utils.meta_cat.ml_utils: precision recall f1-score support\n", + "\n", + " 0 0.64 0.88 0.74 209\n", + " 1 0.21 1.00 0.34 11\n", + " 2 0.98 0.91 0.94 1577\n", + "\n", + " accuracy 0.90 1797\n", + " macro avg 0.61 0.93 0.68 1797\n", + "weighted avg 0.94 0.90 0.92 1797\n", + "\n" + ] + } + ], + "source": [ + "# Perform 2nd round of training\n", + "\n", + "config.model['phase_number'] = 2\n", + "config.train['class_weights'] = [0.3,1,0.05]\n", + "config.train['nepochs'] = 10\n", + "\n", + "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path)" + ] + }, + { + "cell_type": "markdown", + "id": "62771904", + "metadata": {}, + "source": [ + "Using 2 phase learning boosts the performance of the model, especially for the minority classes, wiht 0.88 and 1 recall values.
This highlights the impact of using 2 phase learning for model training with imbalanced datasets\n", + "

NOTE: The observed performance improvements are dataset-dependent, and you may not experience such substantial gains. Additionally, class weights and other hyperparameters will need to be fine-tuned for your specific dataset." + ] + }, + { + "cell_type": "markdown", + "id": "b3d43a3b", + "metadata": {}, + "source": [ + "# Oversampling data" + ] + }, + { + "cell_type": "markdown", + "id": "ca9b70b3", + "metadata": {}, + "source": [ + "You can generate synthetic data to help mitigate class imbalance.
Use this code to generate synthetic data using LLM - [link](https://gist.github.com/shubham-s-agarwal/401ef8bf6cbbd66fa0c76a8fbfc1f6c4)
NOTE: the generated data will require manual quality check to ensure that high quality and relevant data is used for training. " + ] + }, + { + "cell_type": "markdown", + "id": "5835eb2b", + "metadata": {}, + "source": [ + "The data generated from the gist code and the format of the data required by MedCAT are different, requiring manual formatting at the moment. We will update this module to include the code to handle the same." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8161b602", + "metadata": {}, + "outputs": [], + "source": [ + "# To run the training with original + synthetic data\n", + "# Follow all the same steps till initializing the metacat model\n", + "\n", + "# Initialise and train meta_model\n", + "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", + "\n", + "# the format expected is [[['text','of','the','document'], [index of medical entity], \"label\" ],\n", + "# ['text','of','the','document'], [index of medical entity], \"label\" ]]\n", + "\n", + "synthetic_data_export = [[],[],[]]\n", + "\n", + "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path,data_oversampled=synthetic_data_export)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/medcat/2_train_model/2_supervised_training/meta_annotation_training_with_oversampling.ipynb b/medcat/2_train_model/2_supervised_training/meta_annotation_training_with_oversampling.ipynb deleted file mode 100644 index fb43190..0000000 --- a/medcat/2_train_model/2_supervised_training/meta_annotation_training_with_oversampling.ipynb +++ /dev/null @@ -1,80 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "f1ac6acb", - "metadata": {}, - "source": [ - "# Oversampling data" - ] - }, - { - "cell_type": "markdown", - "id": "9a431cf0", - "metadata": {}, - "source": [ - "You can generate synthetic data to help mitigate class imbalance.
Use this code to generate synthetic data using LLM - [link](https://gist.github.com/shubham-s-agarwal/401ef8bf6cbbd66fa0c76a8fbfc1f6c4)
NOTE: the generated data will require manual quality check to ensure that high quality and relevant data is used for training. " - ] - }, - { - "cell_type": "markdown", - "id": "bd8066b5", - "metadata": {}, - "source": [ - "The data generated from the gist code and the format of the data required by MedCAT are different, requiring manual formatting at the moment. We will update this module to include the code to handle the same." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5860552", - "metadata": {}, - "outputs": [], - "source": [ - "# Refer to the meta_annotation_training notebook for the initial steps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "949299e4", - "metadata": {}, - "outputs": [], - "source": [ - "# To run the training with original + synthetic data\n", - "# Follow all the same steps till initializing the metacat model\n", - "\n", - "# Initialise and train meta_model\n", - "mc = MetaCAT(tokenizer=tokenizer, embeddings=None, config=config)\n", - "\n", - "# the format expected is [[['text','of','the','document'], [index of medical entity], \"label\" ],\n", - "# ['text','of','the','document'], [index of medical entity], \"label\" ]]\n", - "\n", - "synthetic_data_export = [[],[],[]]\n", - "\n", - "results = mc.train_from_json(mctrainer_export_path, save_dir_path=save_dir_path,data_oversampled=synthetic_data_export)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}