Upload AI Search.ipynb

tmp/AI Search.ipynb

@@ -0,0 +1,443 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "8495bede-ab8f-416b-b5f2-6a76b1e63935",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Projects\\LLMs\\venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tqdm import tqdm\n",
+    "from sentence_transformers import SentenceTransformer, util"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2b8cae6d-547b-4018-9f68-b0a45284b4b4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# model = SentenceTransformer('DMetaSoul/sbert-chinese-general-v2')\n",
+    "model = SentenceTransformer('TintinMeimei/menglang_yongtulv_aimatch_v1')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "d3907a6f-f8ab-40fe-8702-c8cb81e189c6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def sim(text1, text2):\n",
+    "    emb1 = model.encode(text1, convert_to_tensor=True)\n",
+    "    emb2 = model.encode(text2, convert_to_tensor=True)\n",
+    "    score = util.cos_sim(emb1, emb2)\n",
+    "    return score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "3cec9f05-4ea9-46f8-a393-950c67a0150a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "text1 = '挂机空调'\n",
+    "# text2 = '1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平板电视机、电风扇等家用电器制造。房间空气调节器能效优于《房间空气调节器能效限定值及能效等级》（GB 12021.3）标准1级能效水平；转速可控型房间空气调节器能效优于《转速可控型房间空气调节器能效限定值及能效等级》（GB 21455）标准1级能效水平；多联式空调（热泵）机组能效比优于《多联式空调（热泵）机组能效限定值及能源效率等级》（GB 21454）标准1级能效水平；家用电冰箱能效优于《家用电冰箱耗电量限定值及能效等级》（GB 12021.2）标准1级能效水平；电动洗衣机能效优于《电动洗衣机能效水效限定值及等级》（GB 12021.4）标准1级能效水平；电饭煲能效优于《电饭锅能效限定值及能效等级》（GB 12021.6）标准1级能效水平；平板电视机能效优于《平板电视能效限定值及能效等级》（GB 24850）标准1级能效水平；交流电风扇能效优于《交流电风扇能效限定值及能效等级》（GB 12021.9）标准1级能效水平。其他高效节能家用电器能效均优于相应国家强制性标准1级能效水平。'\n",
+    "# text2 = '包括节能泵、节能型真空干燥设备、节能型真空炉等设备制造。清水离心泵能效指标优于《清水离心泵能效限定值及节能评价值》（GB 19762）标准中节能评价值水平；石油化工离心泵能效优于《石油化工离心泵能效限定值及能效等级》（GB 32284）标准中1级能效水平；潜水电泵能效优于《井用潜水电泵能效限定值及能效等级》（GB 32030）、《小型潜水电泵能效限定值及能效等级》（GB 32029）、《污水污物潜水电泵能效限定值及能效等级》（GB 32031）标准中1级能效水平。'\n",
+    "text2 = '退耕还林'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "d570bf57-2518-4306-a7ae-712e81199460",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[-0.5000]], device='cuda:0')"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sim(text1, text2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "040cc794-9bb0-4c22-986c-933ca55ee637",
+   "metadata": {},
+   "source": [
+    "### Process Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d46e4e74-f7c2-4339-b009-4ba77f1b2f9a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>X1</th>\n",
+       "      <th>X2</th>\n",
+       "      <th>Y</th>\n",
+       "      <th>Split</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>中新制药厂空调末端送回风系统改造-询价公示</td>\n",
+       "      <td>1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>train</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>中新制药厂空调末端送回风系统改造-询价公示</td>\n",
+       "      <td>1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>train</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                      X1                                                 X2  \\\n",
+       "0  中新制药厂空调末端送回风系统改造-询价公示  1.1.11 高效节能家用电器制造\\n包括节能型房间空调器、空调机组、电冰箱、电动洗衣机、平...   \n",
+       "1  中新制药厂空调末端送回风系统改造-询价公示  1.5.1 锅炉（窑炉）节能改造和能效提升\\n包括燃煤锅炉“以大代小”，采用先进燃煤锅炉、节...   \n",
+       "\n",
+       "   Y  Split  \n",
+       "0  1  train  \n",
+       "1  0  train  "
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df_data = pd.read_excel('AI匹配算法样本.xlsx', sheet_name='Sheet1', dtype=str)\n",
+    "df_data.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "673ce0e0-2801-4bb3-8e5d-5c4aff3ac725",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:1: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
+      "  train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n",
+      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:2: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
+      "  eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n",
+      "C:\\Users\\vermouth\\AppData\\Local\\Temp\\ipykernel_22160\\3257358016.py:3: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.\n",
+      "  test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']\n"
+     ]
+    }
+   ],
+   "source": [
+    "train_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='train']\n",
+    "eval_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='eval']\n",
+    "test_data = [item for item in df_data.to_dict(orient='record') if item['Split']=='test']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5037803d-980d-48a1-a61d-528bb9508ce0",
+   "metadata": {},
+   "source": [
+    "### Model 1 - Fine tune a Sentence Transformer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "773429e9-57ce-418f-ad44-3c35d1b31a74",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from sentence_transformers import InputExample, losses\n",
+    "# from torch.utils.data import DataLoader\n",
+    "\n",
+    "# # Prepare data\n",
+    "# train_data_sbert = []\n",
+    "# eval_data_sbert = []\n",
+    "# test_data_sbert = []\n",
+    "\n",
+    "# for item in train_data:\n",
+    "#     label = 1.0 if float(item.get('Y')) == 1 else -1.0\n",
+    "#     train_data_sbert.append(InputExample(texts=[item.get('X1'), item.get('X2')], label=label))\n",
+    "# train_dataloader = DataLoader(train_data_sbert, shuffle=True, batch_size=2)\n",
+    "# train_loss = losses.CosineSimilarityLoss(model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "ec1b68cb-bec3-4896-b196-ec31b1132ad1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from sentence_transformers import evaluation\n",
+    "# evaluator = evaluation.EmbeddingSimilarityEvaluator([item.get('X1') for item in eval_data], [item.get('X2') for item in eval_data], [1.0 if float(item.get('Y'))==1 else -1.0 for item in eval_data])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "7c05c6ef-c5e7-416b-b797-9f8735ae5436",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=1, warmup_steps=100, evaluator=evaluator, evaluation_steps=500)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "7de1e5f0-4b83-4d34-8385-77cdaa0ef08f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model.save('./tmp_model')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fdd686b1-c654-4135-8989-05f23c914afa",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "### Model 2 - No Fine Tune + Threshold Tuning"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a0247889-577d-4a92-8c0f-9c923748df93",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def sim(text1, text2):\n",
+    "    emb1 = model.encode(text1, convert_to_tensor=True)\n",
+    "    emb2 = model.encode(text2, convert_to_tensor=True)\n",
+    "    score = util.cos_sim(emb1, emb2)\n",
+    "    return score\n",
+    "\n",
+    "def _acc_thres(scores, thres):\n",
+    "    correct = 0\n",
+    "    total = len(scores)\n",
+    "    for score, truth in scores:\n",
+    "        truth = float(truth)\n",
+    "        pred = 1 if score >= thres else 0\n",
+    "        if pred == truth:\n",
+    "            correct += 1\n",
+    "    return round(correct/total, 3)\n",
+    "\n",
+    "def model_train(train_data, eval_data):\n",
+    "    score_train = []\n",
+    "    score_eval = []\n",
+    "    for item in tqdm(train_data):\n",
+    "        score = sim(item['X1'], item['X2'])\n",
+    "        score_train.append((score, item['Y']))\n",
+    "    for item in tqdm(eval_data):\n",
+    "        score = sim(item['X1'], item['X2'])\n",
+    "        score_eval.append((score, item['Y']))\n",
+    "    # find threshold that minize train error\n",
+    "    score_train = sorted(score_train, reverse=True)\n",
+    "    win_acc = -1\n",
+    "    win_thres = -1\n",
+    "    for thres in range(5, 100, 5):\n",
+    "        thres = thres*0.01\n",
+    "        acc = _acc_thres(score_train, thres)\n",
+    "        if acc > win_acc:\n",
+    "            win_acc = acc\n",
+    "            win_thres = thres\n",
+    "    eval_acc = _acc_thres(score_eval, win_thres)\n",
+    "    return {'thres': win_thres, 'train_accuracy': win_acc, 'eval_accuracy': eval_acc}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "4e943ef9-ad40-494e-9d53-db9ccbf48bb4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 12256/12256 [13:54<00:00, 14.69it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4248/4248 [04:44<00:00, 14.94it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "r = model_train(train_data, eval_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "9cd38cc9-fe71-45b9-a22e-977a2e787fb5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'thres': 0.25, 'train_accuracy': 0.831, 'eval_accuracy': 0.816}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "53622ff1-7465-4663-a9f0-0c18df37b93e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4468/4468 [04:58<00:00, 14.98it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "score_test = []\n",
+    "for item in tqdm(test_data):\n",
+    "    score = sim(item['X1'], item['X2'])\n",
+    "    score_test.append((score, item['Y']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "47411f71-c774-4274-a1af-2a128589b559",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.815"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "_acc_thres(score_test, r['thres'])\n",
+    "#_acc_thres(score_test, 0.25)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59b741bc-7a20-4ed0-bc9d-b82ec3edff34",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}