diff --git "a/2017/08/14/\347\216\257\345\242\203\345\256\211\350\243\205/\351\243\216\346\240\274\350\277\201\347\247\273\350\275\254\346\215\242\347\232\204\344\270\244\347\247\215\345\256\236\347\216\260/index.html" "b/2017/08/14/\347\216\257\345\242\203\345\256\211\350\243\205/\351\243\216\346\240\274\350\277\201\347\247\273\350\275\254\346\215\242\347\232\204\344\270\244\347\247\215\345\256\236\347\216\260/index.html"
index 742b7a76..566e082d 100644
--- "a/2017/08/14/\347\216\257\345\242\203\345\256\211\350\243\205/\351\243\216\346\240\274\350\277\201\347\247\273\350\275\254\346\215\242\347\232\204\344\270\244\347\247\215\345\256\236\347\216\260/index.html"
+++ "b/2017/08/14/\347\216\257\345\242\203\345\256\211\350\243\205/\351\243\216\346\240\274\350\277\201\347\247\273\350\275\254\346\215\242\347\232\204\344\270\244\347\247\215\345\256\236\347\216\260/index.html"
@@ -31,8 +31,8 @@
 <meta property="article:modified_time" content="2024-06-15T08:01:01.811Z">
 <meta property="article:author" content="wnma3mz">
 <meta property="article:tag" content="Python">
-<meta property="article:tag" content="Tensorflow">
 <meta property="article:tag" content="Caffe">
+<meta property="article:tag" content="Tensorflow">
 <meta property="article:tag" content="风格迁移">
 <meta name="twitter:card" content="summary">
 
@@ -430,8 +430,8 @@ <h2 id="总结">总结</h2>
       <footer class="post-footer">
           <div class="post-tags">
               <a href="/hexo_blog/tags/Python/" rel="tag"># Python</a>
-              <a href="/hexo_blog/tags/Tensorflow/" rel="tag"># Tensorflow</a>
               <a href="/hexo_blog/tags/Caffe/" rel="tag"># Caffe</a>
+              <a href="/hexo_blog/tags/Tensorflow/" rel="tag"># Tensorflow</a>
               <a href="/hexo_blog/tags/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB/" rel="tag"># 风格迁移</a>
           </div>
 
diff --git "a/2021/10/30/\347\216\257\345\242\203\345\256\211\350\243\205/Ubuntu20.04\345\256\211\350\243\205\346\267\261\345\272\246\345\255\246\344\271\240\347\216\257\345\242\203\345\217\212\351\205\215\347\275\256\344\270\252\344\272\272\347\216\257\345\242\203\345\260\217\350\256\260/index.html" "b/2021/10/30/\347\216\257\345\242\203\345\256\211\350\243\205/Ubuntu20.04\345\256\211\350\243\205\346\267\261\345\272\246\345\255\246\344\271\240\347\216\257\345\242\203\345\217\212\351\205\215\347\275\256\344\270\252\344\272\272\347\216\257\345\242\203\345\260\217\350\256\260/index.html"
index 7db84b57..da74a4b9 100644
--- "a/2021/10/30/\347\216\257\345\242\203\345\256\211\350\243\205/Ubuntu20.04\345\256\211\350\243\205\346\267\261\345\272\246\345\255\246\344\271\240\347\216\257\345\242\203\345\217\212\351\205\215\347\275\256\344\270\252\344\272\272\347\216\257\345\242\203\345\260\217\350\256\260/index.html"
+++ "b/2021/10/30/\347\216\257\345\242\203\345\256\211\350\243\205/Ubuntu20.04\345\256\211\350\243\205\346\267\261\345\272\246\345\255\246\344\271\240\347\216\257\345\242\203\345\217\212\351\205\215\347\275\256\344\270\252\344\272\272\347\216\257\345\242\203\345\260\217\350\256\260/index.html"
@@ -32,8 +32,8 @@
 <meta property="article:author" content="wnma3mz">
 <meta property="article:tag" content="CUDA">
 <meta property="article:tag" content="PyTorch">
-<meta property="article:tag" content="Ubuntu">
 <meta property="article:tag" content="Tensorflow">
+<meta property="article:tag" content="Ubuntu">
 <meta property="article:tag" content="CUDNN">
 <meta name="twitter:card" content="summary">
 
@@ -333,8 +333,8 @@ <h4 id="远程连接">远程连接</h4>
           <div class="post-tags">
               <a href="/hexo_blog/tags/CUDA/" rel="tag"># CUDA</a>
               <a href="/hexo_blog/tags/PyTorch/" rel="tag"># PyTorch</a>
-              <a href="/hexo_blog/tags/Ubuntu/" rel="tag"># Ubuntu</a>
               <a href="/hexo_blog/tags/Tensorflow/" rel="tag"># Tensorflow</a>
+              <a href="/hexo_blog/tags/Ubuntu/" rel="tag"># Ubuntu</a>
               <a href="/hexo_blog/tags/CUDNN/" rel="tag"># CUDNN</a>
           </div>
 
diff --git a/css/main.css b/css/main.css
index 8291b7a9..dbe9ca11 100644
--- a/css/main.css
+++ b/css/main.css
@@ -1165,7 +1165,7 @@ pre .javascript .function {
 }
 .links-of-author a::before,
 .links-of-author span.exturl::before {
-  background: #280d90;
+  background: #e987b5;
   border-radius: 50%;
   content: ' ';
   display: inline-block;
diff --git a/search.xml b/search.xml
index 180e5d82..48a1cd9b 100644
--- a/search.xml
+++ b/search.xml
@@ -497,375 +497,47 @@ one-shot 编码是 [0, 1, 0] 和 [0, 0, 1] 。</p>
       </tags>
   </entry>
   <entry>
-    <title>LLM的推理相关计算公式</title>
-    <url>/hexo_blog/2024/09/05/LLM%E6%8E%A8%E7%90%86%E7%9B%B8%E5%85%B3%E8%AE%A1%E7%AE%97%E5%85%AC%E5%BC%8F/</url>
-    <content><![CDATA[<p>记录 LLM 在推理上的理论计算公式</p>
-<span id="more"></span>
-<h3 id="参数量">参数量</h3>
-<p>embedding + lm head + transformer block <span
-class="math inline">\(\times\)</span> N</p>
-<ul>
-<li>Embedding / LM Head 的参数量：vocab_size <span
-class="math inline">\(\times\)</span> hidden_size</li>
-<li>transformer block 的参数量
-<ul>
-<li>self-attention：qkvo 四个线性层，需要分别考虑两种 Attention
-<ul>
-<li>multi-head attention（MHA）：4 <span
-class="math inline">\(\times\)</span> hidden_size <span
-class="math inline">\(^2\)</span></li>
-<li>grouped query attention（GQA）：
-<ul>
-<li><p>q 和 o，2 $$ hidden_size <span
-class="math inline">\(^2\)</span></p></li>
-<li><p>k 和 v，2 $$ hidden_size <span class="math inline">\(^2\)</span>
-/ num_attention_heads $$ num_key_value_heads</p></li>
-</ul></li>
-</ul></li>
-<li>FFN：三个线性层，3 <span class="math inline">\(\times\)</span>
-hidden_size <span class="math inline">\(\times\)</span>
-intermediate_size</li>
-<li>layer norm：2 <span class="math inline">\(\times\)</span> 2 <span
-class="math inline">\(\times\)</span> hidden_size</li>
-</ul></li>
-</ul>
-<p>以 <a
-href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
-为例，关键参数</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>由于 num_attention_heads == num_key_value_heads，所以是 MHA</p>
-<ul>
-<li>Embedding/LM Head： <span class="math inline">\(32000\times
-5120\)</span></li>
-<li>transformer block
-<ul>
-<li>self-attention：<span class="math inline">\(4\times 5120\times
-5120\)</span></li>
-<li>FFN: <span class="math inline">\(3\times 5120\times
-13824\)</span></li>
-<li>layer norm: <span class="math inline">\(4\times 5120\)</span></li>
-</ul></li>
-</ul>
-<p>综上，列出计算</p>
-<p><span class="math display">\[2\times 32000\times 5120 + 40\times
-(4\times 5120\times 5120 + 3\times 5120\times 13824 + 4\times 5120) =
-13,016,268,800 \approx 13B\]</span></p>
-<p>如果是 (b)float16，则占用空间/显存为 13 $$ 2 = 26G</p>
-<p>如果是 GQA，self-attn 参数量公式为</p>
-<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
-<p><span class="math inline">\(4\times b\times l\times num\_heads\times
-embed\_size\_per\_head \times (s+n)\)</span></p>
-<p>参数说明</p>
-<ul>
-<li>b: 句子条数</li>
-<li>l：层数</li>
-<li>num_heads：隐层大小 (num_key_value_heads)</li>
-<li>embed_size_per_head：每个头的大小 (hidden_size /
-num_attention_heads)</li>
-<li>s：输入长度</li>
-<li>n：输出长度</li>
-<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
-</ul>
-<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
-<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
-<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
-<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
-80 / 1024 / 1024 = 160\)</span> M</p>
-<p>简单来说，每增加一个 token</p>
-<p>8B 就会增加 0.125 M 的显存</p>
-<p>70B 就会增加 0.3125 M 的显存</p>
-<h3 id="计算量">计算量</h3>
-<p>Embedding 可以视作一个哈希表，没有计算量</p>
-<ul>
-<li>LM Head：<span class="math inline">\(2 \times b \times s\times
-hidden\_size\times V\)</span></li>
-<li>Self-attention：
-<ul>
-<li>q_proj: <span class="math inline">\(2 \times b \times s \times
-hidden\_size^2\)</span></li>
-<li>k_proj和v_proj:
-<ul>
-<li>MHA: <span class="math inline">\(2\times(2 \times b \times s \times
-hidden\_size^2)\)</span></li>
-<li>GQA: <span class="math inline">\(2\times(2 \times b \times s \times
-hidden\_size^2 / (num\_attention\_heads
-\times  num\_key\_value\_heads)^2 )\)</span></li>
-</ul></li>
-<li>attn_weights: <span class="math inline">\(2 \times b \times s^2
-\times hidden\_size\)</span></li>
-<li>attn_output: <span class="math inline">\(2 \times b \times s^2
-\times hidden\_size\)</span></li>
-<li>o_proj: <span class="math inline">\(2 \times b \times s \times
-hidden\_size^2\)</span></li>
-</ul></li>
-<li>FFN:
-<ul>
-<li>gate_proj 和 up_proj: <span class="math inline">\(2\times 2\times b
-\times s\times (hidden\_size\times intermediate\_size)\)</span></li>
-<li>down_proj: <span class="math inline">\(2\times b \times s\times
-(intermediate\_size\times hidden\_size)\)</span></li>
-</ul></li>
-</ul>
-<p>参数说明 - b: 句子条数 - s: 输入长度 - 2:
-一次乘法操作和一次加法操作</p>
-<p>还是以 <a
-href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
-为例，关键参数</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>假设 b=1，s=1，则整体计算量有</p>
-<ul>
-<li>LM Head: <span class="math inline">\(2\times 5120\times
-32000\)</span></li>
-<li>self-attention:
-<ul>
-<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
-<li><span class="math inline">\(4\times 5120\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
-</ul></li>
-<li>MLP:
-<ul>
-<li><span class="math inline">\(4\times 5120\times 13824\)</span></li>
-<li><span class="math inline">\(2\times 13824\times 5120\)</span></li>
-</ul></li>
-</ul>
-<p>模型有 40 层，所以</p>
-<p><span class="math inline">\(2\times 5120\times 32000+
-40\times(8\times 5120\times 5120+4\times 5120+ 2\times 5120\times
-13824+2\times 13824\times 5120) = 20,041,728,000\)</span></p>
-<p>参考资料：</p>
-<p>https://blog.csdn.net/wxc971231/article/details/135434478</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>Inference</tag>
-        <tag>Math</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>LLM 深度的”稀疏性“</title>
-    <url>/hexo_blog/2024/03/28/LLM%E6%B7%B1%E5%BA%A6%E7%9A%84%E7%A8%80%E7%96%8F%E6%80%A7/</url>
-    <content><![CDATA[<p>LLM 在宽度上的稀疏性可以通过 MoE
-这种架构来优化。而深度上也具备稀疏性，故而可以进行裁剪。</p>
+    <title>LLM 不同硬件推理速度对比</title>
+    <url>/hexo_blog/2024/07/21/LLM%20%E7%A1%AC%E4%BB%B6%E6%AF%94%E4%BB%B7/</url>
+    <content><![CDATA[<p>对比不同硬件下，LLM 的推理速度</p>
 <span id="more"></span>
-<h2 id="背景">背景</h2>
-<p>这篇<a
-href="https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens">博客</a>发现到
-GPT-2 在深度上的输出具有相似性。即，第 20 层的输出接入 LM Head 和第 21
-层的输出接入 LM Head 是相似，甚至一致的。</p>
-<p>故而，模型在深度上可能是稀疏的。所以可以按 block
-的维度进行裁剪。如下图所示，</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/image-2024-3-28_18-23-0.png" /></p>
-<p>找到三篇相似的工作</p>
-<ul>
-<li>上交：<a href="https://arxiv.org/html/2402.11187v1">LaCo: Large
-Language Model Pruning via Layer Collapse</a></li>
-<li>Meta：<a href="https://arxiv.org/html/2403.17887">The Unreasonable
-Ineffectiveness of the Deeper Layers</a></li>
-<li>百川：<a href="https://arxiv.org/html/2403.03853v2">ShortGPT: Layers
-in Large Language Models are More Redundant Than You Expect</a></li>
-</ul>
-<p>越靠近模型输出的那些层相似度较高，可以通过删除一些“冗余”层来减小模型参数</p>
-<h2 id="方法的共通性">方法的共通性</h2>
-<ul>
-<li>构建一个“中性”数据集！！！</li>
-<li>观察不同层的输入/输出</li>
-<li>用某种度量距离的方法测量不同层是否相似
-<ul>
-<li>百川：Block Influence</li>
-<li>Meta：角距离</li>
-</ul></li>
-</ul>
-<h2 id="meta-论文结果">Meta 论文结果</h2>
-<h3 id="不同模型具有相似的表现">不同模型具有相似的表现</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x2.png" /></p>
-<p>注：</p>
-<ul>
-<li>实线（healing）：扔掉一些层之后进行了训练</li>
-<li>虚线（no healing）： → 扔掉这些层后不训练</li>
-<li>实线和虚线扔掉的是相同的层</li>
-<li>横坐标是扔掉的比例</li>
-</ul>
-<p>Q：MMLU/BoolQ 指标具备足够的可信度？</p>
-<p>A：个人感觉没有，但有后续有其他的一些实验支撑</p>
-<p>Q：Qwen 为啥表现不大一样？</p>
-<p>A：论文其他部分进行一定的解释，但与本文的重点相关性不强，所以不展开介绍</p>
-<h3 id="暴力裁剪-vs-提出的方法">暴力裁剪 vs 提出的方法</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x5.png" /></p>
-<p>有一种直接粗暴的方法就是直接扔掉最后若干层，这样就不需要数据集啥的来选层。</p>
-<p>奇怪的现象就是直接扔掉最后若干层，指标会出现先降低再提升（不训练的情况）。</p>
-<h2 id="百川论文的结论">百川论文的结论</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/baichuan.png" /></p>
-<p><strong>第一层和最后一层很重要！！！</strong></p>
-<h2 id="剪枝比率">剪枝比率</h2>
-<p>利用宽度稀疏性，一种优化架构 MoE
-可以做到降低一半的参数且保持性能不变。</p>
-<p>而在深度稀疏上，这一点是比较难做到的。</p>
-<table>
+<p>生成阶段，20 Token/s 认为是一个比较可以接受的速度，即 20 / 1000 =
+0.02 s = 20 ms</p>
+<table style="width:100%;">
+<colgroup>
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+</colgroup>
 <thead>
 <tr>
+<th>设备</th>
 <th>模型尺寸</th>
-<th>层数</th>
-<th>参数量</th>
+<th>速度</th>
+<th>来源</th>
+<th>记录时间</th>
+<th>备注</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>70B</td>
-<td>80</td>
-<td>69,434,351,616</td>
-</tr>
-<tr>
-<td>70B</td>
-<td>40</td>
-<td>35,208,175,616</td>
+<td>1 x RasPi 5 8 GB</td>
+<td>Llama 3 8B （Q4）</td>
+<td>564.31 ms, 1.77 t/s</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>70B</td>
-<td>45</td>
-<td>39,486,447,616</td>
-</tr>
-<tr>
-<td>34B</td>
-<td>48</td>
-<td>34,201,673,728</td>
-</tr>
-<tr>
-<td>34B</td>
-<td>24</td>
-<td>17,591,836,672</td>
-</tr>
-</tbody>
-</table>
-<p>简单来说，对于 70B 模型要剪枝到 34B，需要砍掉一半的层；对于 34B
-模型要剪枝到 13B，要砍掉大于一半的层。</p>
-<p>通过 Meta 论文的实验结果显示，当裁剪超过
-40%，就会有较大的风险不能恢复模型性能。</p>
-<h2 id="实验">实验</h2>
-<p>在 boolq/gsm8k
-数据集上简单了进行若干实验，这两个数据集均具备训练集和评测集。所以，对
-13B 模型进行暴力裁剪后（～40%
-参数）在训练集上进行训练，在评测集上进行评测。</p>
-<ul>
-<li>裁剪方法：保留最后一层，从最后一层往前数，直至裁剪到 40% 为止。</li>
-<li>完整训练集：对应评测的训练集+通用的QA/对话数据集</li>
-</ul>
-<h3 id="gsm8k">gsm8k</h3>
-<p>是一个考验数学/逻辑推理能力的评测结果</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold 48 clips in April.\nThen she sold half as many in May, which is 48 / 2 = 24.\nSo altogether she sold 48 + 24 = 72 clips.\nThe answer is 72.&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>在该评测上指标直接崩掉，如原来可能是 &gt;70，现在只有 2%</p>
-<table>
-<colgroup>
-<col style="width: 47%" />
-<col style="width: 52%" />
-</colgroup>
-<thead>
-<tr>
-<th><strong>prompt</strong></th>
-<th><strong>pred</strong></th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>Janet’s ducks lay 16 eggs per day. She eats three for breakfast
-every morning and bakes muffins for her friends every day with four. She
-sells the remainder at the farmers' market daily for $2 per fresh duck
-egg. How much in dollars does she make every day at the farmers' market?
-Let's think step by step.</td>
-<td>Each egg is 16 eggs/day for 16 eggs/day for 1600000000 / 16 =
-128000000. So, the total number of eggs is 128000000. The total number
-of eggs is 160000000 - 128000000 = 128000000.The answer is
-128000000.</td>
-</tr>
-<tr>
-<td>A robe takes 2 bolts of blue fiber and half that much white fiber.
-How many bolts in total does it take? Let's think step by step</td>
-<td>The rose is 2 bolts of blue fibers. Half that amount is 2 / 2 = 1.
-So the rose is 2 - 1 = 2 flobs. The answer is 2.</td>
-</tr>
-<tr>
-<td>Josh decides to try flipping a house. He buys a house for $80,000
-and then puts in $50,000 in repairs. This increased the value of the
-house by 150%. How much profit did he make? Let's think step by
-step.</td>
-<td>The house was originally 80,000 dollars. The new house is 80,000 +
-50,000 = 15,000 dollars. The new house is 15,000 dollars. So, the total
-value of the house is 15,000 + 80,000 + 15,000 = 11,000 dollars. The
-total value of the house is 11,000 dollars. The total value of the house
-is 11,000 + 15,000 = 11,000 + 15,000 = 11,000 + 15,000 =</td>
-</tr>
-</tbody>
-</table>
-<h3 id="boolq">boolq</h3>
-<p>而 boolq 相对是一个简单的数据集，输出只需要有 yes or no 即可。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Passage: Persian (/ˈpɜːrʒən, -ʃən/), also known by its endonym Farsi (فارسی fārsi (fɒːɾˈsiː) ( listen)), is one of the Western Iranian languages within the Indo-Iranian branch of the Indo-European language family. It is primarily spoken in Iran, Afghanistan (officially known as Dari since 1958), and Tajikistan (officially known as Tajiki since the Soviet era), and some other regions which historically were Persianate societies and considered part of Greater Iran. It is written in the Persian alphabet, a modified variant of the Arabic script, which itself evolved from the Aramaic alphabet.\nQuestion: do iran and afghanistan speak the same language\nAnswer:&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Yes&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span></span><br></pre></td></tr></table></figure>
-<p>在该指标上结果就是相对正常的，符合 Meta 论文的结果。</p>
-<ul>
-<li>裁剪前：～84%</li>
-<li>裁剪后：～68%</li>
-<li>裁剪+训练后：～85%</li>
-</ul>
-<p>所以，模型深度或许是加强模型推理能力的关键因素之一？</p>
-]]></content>
-      <categories>
-        <category>PaperReading</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>LLM</tag>
-        <tag>knowledge distillation</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>LLM 不同硬件推理速度对比</title>
-    <url>/hexo_blog/2024/07/21/LLM%20%E7%A1%AC%E4%BB%B6%E6%AF%94%E4%BB%B7/</url>
-    <content><![CDATA[<p>对比不同硬件下，LLM 的推理速度</p>
-<span id="more"></span>
-<p>生成阶段，20 Token/s 认为是一个比较可以接受的速度，即 20 / 1000 =
-0.02 s = 20 ms</p>
-<table style="width:100%;">
-<colgroup>
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-</colgroup>
-<thead>
-<tr>
-<th>设备</th>
-<th>模型尺寸</th>
-<th>速度</th>
-<th>来源</th>
-<th>记录时间</th>
-<th>备注</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>1 x RasPi 5 8 GB</td>
-<td>Llama 3 8B （Q4）</td>
-<td>564.31 ms, 1.77 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 556.67 ms, T: 6.17 ms</td>
-<td>https://github.com/b4rtaz/distributed-llama/blob/main/README.md</td>
-<td>2024.07.21</td>
-<td></td>
-<td></td>
-<td></td>
+<td>I: 556.67 ms, T: 6.17 ms</td>
+<td>https://github.com/b4rtaz/distributed-llama/blob/main/README.md</td>
+<td>2024.07.21</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
 <td>4 x RasPi 5 8 GB</td>
@@ -1154,28 +826,61 @@ is 11,000 + 15,000 = 11,000 + 15,000 = 11,000 + 15,000 =</td>
       </tags>
   </entry>
   <entry>
-    <title>LLM的推理加速</title>
-    <url>/hexo_blog/2024/06/15/LLM%E7%9A%84%E6%8E%A8%E7%90%86%E5%8A%A0%E9%80%9F/</url>
-    <content><![CDATA[<p>对于两种场景下介绍 LLM 的 推理加速方法</p>
-<p>长输入，短输出</p>
-<p>长输出但输出 token 有限</p>
+    <title>LLM的推理相关计算公式</title>
+    <url>/hexo_blog/2024/09/05/LLM%E6%8E%A8%E7%90%86%E7%9B%B8%E5%85%B3%E8%AE%A1%E7%AE%97%E5%85%AC%E5%BC%8F/</url>
+    <content><![CDATA[<p>记录 LLM 在推理上的理论计算公式</p>
 <span id="more"></span>
-<p>大语言模型在进行推理时是非常耗时的，因为需要对每个 token
-进行计算。对于推理加速这个话题，在工程上可以从很多方面考虑：</p>
+<h3 id="参数量">参数量</h3>
+<p>embedding + lm head + transformer block <span
+class="math inline">\(\times\)</span> N</p>
 <ul>
-<li>模型进行量化/剪枝，降低模型浮点计算成本</li>
-<li>Continuous Batch，批量计算 Next Token</li>
-<li>KV Cache：LLM 推理的是 Token by Token
-生成，每次生成的时候会计算所有输入 Token 的
-Attention，故可以缓存已计算的 Attention，减少计算量</li>
-<li>对显存进行管理，如 <a
-href="https://wnma3mz.github.io/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/">PageAttention</a></li>
-<li>KV Cache 量化</li>
-<li>...</li>
+<li>Embedding / LM Head 的参数量：vocab_size <span
+class="math inline">\(\times\)</span> hidden_size</li>
+<li>transformer block 的参数量
+<ul>
+<li>self-attention：qkvo 四个线性层，需要分别考虑两种 Attention
+<ul>
+<li>multi-head attention（MHA）：4 <span
+class="math inline">\(\times\)</span> hidden_size <span
+class="math inline">\(^2\)</span></li>
+<li>grouped query attention（GQA）：
+<ul>
+<li><p>q 和 o，2 $$ hidden_size <span
+class="math inline">\(^2\)</span></p></li>
+<li><p>k 和 v，2 $$ hidden_size <span class="math inline">\(^2\)</span>
+/ num_attention_heads $$ num_key_value_heads</p></li>
+</ul></li>
+</ul></li>
+<li>FFN：三个线性层，3 <span class="math inline">\(\times\)</span>
+hidden_size <span class="math inline">\(\times\)</span>
+intermediate_size</li>
+<li>layer norm：2 <span class="math inline">\(\times\)</span> 2 <span
+class="math inline">\(\times\)</span> hidden_size</li>
+</ul></li>
 </ul>
-<p>而其他在工程上的优化，往往是针对特定场景的，下面介绍两种场景下的优化方法。</p>
-<h2 id="prefix-cache长输入短输出">Prefix Cache：长输入，短输出</h2>
-<p>注：优化显存并不会减少计算量，只是减少了显存占用，从而提升了吞吐。</p>
+<p>以 <a
+href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
+为例，关键参数</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>由于 num_attention_heads == num_key_value_heads，所以是 MHA</p>
+<ul>
+<li>Embedding/LM Head： <span class="math inline">\(32000\times
+5120\)</span></li>
+<li>transformer block
+<ul>
+<li>self-attention：<span class="math inline">\(4\times 5120\times
+5120\)</span></li>
+<li>FFN: <span class="math inline">\(3\times 5120\times
+13824\)</span></li>
+<li>layer norm: <span class="math inline">\(4\times 5120\)</span></li>
+</ul></li>
+</ul>
+<p>综上，列出计算</p>
+<p><span class="math display">\[2\times 32000\times 5120 + 40\times
+(4\times 5120\times 5120 + 3\times 5120\times 13824 + 4\times 5120) =
+13,016,268,800 \approx 13B\]</span></p>
+<p>如果是 (b)float16，则占用空间/显存为 13 $$ 2 = 26G</p>
+<p>如果是 GQA，self-attn 参数量公式为</p>
 <h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
 <p><span class="math inline">\(4\times b\times l\times num\_heads\times
 embed\_size\_per\_head \times (s+n)\)</span></p>
@@ -1198,779 +903,787 @@ num_attention_heads)</li>
 <p>简单来说，每增加一个 token</p>
 <p>8B 就会增加 0.125 M 的显存</p>
 <p>70B 就会增加 0.3125 M 的显存</p>
-<p>这个显存占有直到整个句子生成完成之后才会释放，所以对于长句子，KV
-Cache 的显存占用是非常大的。</p>
-<p>显存决定了整体的吞吐量。对于 70B，假设有 1k 个请求，每个请求的总
-token 为 1k，那么显存占用为 0.3125M * 1000 * 1000 = 0.3G。</p>
-<h3 id="方法">方法</h3>
-<p>在 LLM
-推理时，往往是用对话的方式进行推理，而每次对话的开头可能是相同（System），这一部分的
-Attention 是不变的，所以可以缓存这部分 Attention。</p>
-<p>加速的原因：</p>
+<h3 id="计算量">计算量</h3>
+<p>Embedding 可以视作一个哈希表，没有计算量</p>
 <ul>
-<li>prefix 的 kv cache 不需要计算</li>
-<li>prefix 的 kv cache 复用节约了显存，提升吞吐（主要收益）</li>
+<li>LM Head：<span class="math inline">\(2 \times b \times s\times
+hidden\_size\times V\)</span></li>
+<li>Self-attention：
+<ul>
+<li>q_proj: <span class="math inline">\(2 \times b \times s \times
+hidden\_size^2\)</span></li>
+<li>k_proj和v_proj:
+<ul>
+<li>MHA: <span class="math inline">\(2\times(2 \times b \times s \times
+hidden\_size^2)\)</span></li>
+<li>GQA: <span class="math inline">\(2\times(2 \times b \times s \times
+hidden\_size^2 / (num\_attention\_heads
+\times  num\_key\_value\_heads)^2 )\)</span></li>
+</ul></li>
+<li>attn_weights: <span class="math inline">\(2 \times b \times s^2
+\times hidden\_size\)</span></li>
+<li>attn_output: <span class="math inline">\(2 \times b \times s^2
+\times hidden\_size\)</span></li>
+<li>o_proj: <span class="math inline">\(2 \times b \times s \times
+hidden\_size^2\)</span></li>
+</ul></li>
+<li>FFN:
+<ul>
+<li>gate_proj 和 up_proj: <span class="math inline">\(2\times 2\times b
+\times s\times (hidden\_size\times intermediate\_size)\)</span></li>
+<li>down_proj: <span class="math inline">\(2\times b \times s\times
+(intermediate\_size\times hidden\_size)\)</span></li>
+</ul></li>
 </ul>
-<p>所以适用场景的一个明显特点就是必须要有长的 system prefix，而且这个
-prefix
-是不变的。另外，如果模型输出过长的话，这个优化的收益就会变小。因为它并不会加速在生成过程中的计算，只是减少了显存占用。</p>
-<h3 id="收益">收益</h3>
-<p>https://github.com/vllm-project/vllm/issues/227</p>
-<p>For each request, the prefix length is 200, the input length is 30,
-and the output length is 50.</p>
-<table>
-<colgroup>
-<col style="width: 12%" />
-<col style="width: 15%" />
-<col style="width: 20%" />
-<col style="width: 29%" />
-<col style="width: 20%" />
-</colgroup>
+<p>参数说明 - b: 句子条数 - s: 输入长度 - 2:
+一次乘法操作和一次加法操作</p>
+<p>还是以 <a
+href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
+为例，关键参数</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>假设 b=1，s=1，则整体计算量有</p>
+<ul>
+<li>LM Head: <span class="math inline">\(2\times 5120\times
+32000\)</span></li>
+<li>self-attention:
+<ul>
+<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
+<li><span class="math inline">\(4\times 5120\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
+</ul></li>
+<li>MLP:
+<ul>
+<li><span class="math inline">\(4\times 5120\times 13824\)</span></li>
+<li><span class="math inline">\(2\times 13824\times 5120\)</span></li>
+</ul></li>
+</ul>
+<p>模型有 40 层，所以</p>
+<p><span class="math inline">\(2\times 5120\times 32000+
+40\times(8\times 5120\times 5120+4\times 5120+ 2\times 5120\times
+13824+2\times 13824\times 5120) = 20,041,728,000\)</span></p>
+<p>参考资料：</p>
+<p>https://blog.csdn.net/wxc971231/article/details/135434478</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>NLP</tag>
+        <tag>Inference</tag>
+        <tag>Math</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>LLM 深度的”稀疏性“</title>
+    <url>/hexo_blog/2024/03/28/LLM%E6%B7%B1%E5%BA%A6%E7%9A%84%E7%A8%80%E7%96%8F%E6%80%A7/</url>
+    <content><![CDATA[<p>LLM 在宽度上的稀疏性可以通过 MoE
+这种架构来优化。而深度上也具备稀疏性，故而可以进行裁剪。</p>
+<span id="more"></span>
+<h2 id="背景">背景</h2>
+<p>这篇<a
+href="https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens">博客</a>发现到
+GPT-2 在深度上的输出具有相似性。即，第 20 层的输出接入 LM Head 和第 21
+层的输出接入 LM Head 是相似，甚至一致的。</p>
+<p>故而，模型在深度上可能是稀疏的。所以可以按 block
+的维度进行裁剪。如下图所示，</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/image-2024-3-28_18-23-0.png" /></p>
+<p>找到三篇相似的工作</p>
+<ul>
+<li>上交：<a href="https://arxiv.org/html/2402.11187v1">LaCo: Large
+Language Model Pruning via Layer Collapse</a></li>
+<li>Meta：<a href="https://arxiv.org/html/2403.17887">The Unreasonable
+Ineffectiveness of the Deeper Layers</a></li>
+<li>百川：<a href="https://arxiv.org/html/2403.03853v2">ShortGPT: Layers
+in Large Language Models are More Redundant Than You Expect</a></li>
+</ul>
+<p>越靠近模型输出的那些层相似度较高，可以通过删除一些“冗余”层来减小模型参数</p>
+<h2 id="方法的共通性">方法的共通性</h2>
+<ul>
+<li>构建一个“中性”数据集！！！</li>
+<li>观察不同层的输入/输出</li>
+<li>用某种度量距离的方法测量不同层是否相似
+<ul>
+<li>百川：Block Influence</li>
+<li>Meta：角距离</li>
+</ul></li>
+</ul>
+<h2 id="meta-论文结果">Meta 论文结果</h2>
+<h3 id="不同模型具有相似的表现">不同模型具有相似的表现</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x2.png" /></p>
+<p>注：</p>
+<ul>
+<li>实线（healing）：扔掉一些层之后进行了训练</li>
+<li>虚线（no healing）： → 扔掉这些层后不训练</li>
+<li>实线和虚线扔掉的是相同的层</li>
+<li>横坐标是扔掉的比例</li>
+</ul>
+<p>Q：MMLU/BoolQ 指标具备足够的可信度？</p>
+<p>A：个人感觉没有，但有后续有其他的一些实验支撑</p>
+<p>Q：Qwen 为啥表现不大一样？</p>
+<p>A：论文其他部分进行一定的解释，但与本文的重点相关性不强，所以不展开介绍</p>
+<h3 id="暴力裁剪-vs-提出的方法">暴力裁剪 vs 提出的方法</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x5.png" /></p>
+<p>有一种直接粗暴的方法就是直接扔掉最后若干层，这样就不需要数据集啥的来选层。</p>
+<p>奇怪的现象就是直接扔掉最后若干层，指标会出现先降低再提升（不训练的情况）。</p>
+<h2 id="百川论文的结论">百川论文的结论</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/baichuan.png" /></p>
+<p><strong>第一层和最后一层很重要！！！</strong></p>
+<h2 id="剪枝比率">剪枝比率</h2>
+<p>利用宽度稀疏性，一种优化架构 MoE
+可以做到降低一半的参数且保持性能不变。</p>
+<p>而在深度稀疏上，这一点是比较难做到的。</p>
+<table>
 <thead>
 <tr>
-<th>Load (QPS)</th>
-<th>Method</th>
-<th>Requests/s</th>
-<th>Average Latency per Req</th>
-<th>First Token Time</th>
+<th>模型尺寸</th>
+<th>层数</th>
+<th>参数量</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>10 QPS</td>
-<td>Prefix Cache</td>
-<td>9.83 requests/s</td>
-<td>1.97 s</td>
-<td>0.29 s</td>
-</tr>
-<tr>
-<td>10 QPS</td>
-<td>Base</td>
-<td>9.80 requests/s</td>
-<td>2.87 s</td>
-<td>0.45 s</td>
+<td>70B</td>
+<td>80</td>
+<td>69,434,351,616</td>
 </tr>
 <tr>
-<td>15 QPS</td>
-<td>Prefix Cache</td>
-<td>14.30 requests/s</td>
-<td>2.98 s</td>
-<td>0.39 s</td>
+<td>70B</td>
+<td>40</td>
+<td>35,208,175,616</td>
 </tr>
 <tr>
-<td>15 QPS</td>
-<td>Base</td>
-<td>13.24 requests/s</td>
-<td>8.65 s</td>
-<td>1.02 s</td>
+<td>70B</td>
+<td>45</td>
+<td>39,486,447,616</td>
 </tr>
 <tr>
-<td>25 QPS</td>
-<td>Prefix Cache</td>
-<td>19.81 requests/s</td>
-<td>6.46 s</td>
-<td>0.84 s</td>
+<td>34B</td>
+<td>48</td>
+<td>34,201,673,728</td>
 </tr>
 <tr>
-<td>25 QPS</td>
-<td>Base</td>
-<td>14.08 requests/s</td>
-<td>13.67 s</td>
-<td>4.74 s</td>
+<td>34B</td>
+<td>24</td>
+<td>17,591,836,672</td>
 </tr>
 </tbody>
 </table>
-<h2 id="并行解码长输出但输出-token-有限">并行解码：长输出但输出 token
-有限</h2>
-<p>LLM 的推理是 Token by Token 生成的，如果有一种方式可以每次生成多个
-token，那么就可以提升推理速度。</p>
-<h3 id="方法-1">方法</h3>
-<p>以下图为例，对 LLM 的模型结构进行一定的修改，使每次可以同时生成 3 个
-token。</p>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/llm_speedup/251e873bb1322b4d31fee19f0c81c424.png"
-alt="并行解码" />
-<figcaption aria-hidden="true">并行解码</figcaption>
-</figure>
-<p>并行解码的方式不一定要对结构进行修改，这里只是一种实现方式。</p>
-<p>具体实现方式： -
-训练：修改结构后对模型的新增部分进行常规的训练，其他部分冻结，减少训练成本
-- 推理：模型生成的时候会把连续 3 个 token 的 top-K
-的可能均生成，交由原模型进行打分，然后选择最优的一个。</p>
-<h3 id="问题">问题</h3>
-<p>并行生成的 Token 的可靠性是取决于具体场景的。如果生成的 Token
-可能性只有 20 个，那么并行生成的 Token
-的可靠性就会相对较高，这种对于模型而言是更容易学习的。但如果生成的 Token
-可能有 10000 个，那么并行生成的 Token 的可靠性就会相对较低。</p>
-<p>换而言之，Token
-可能性越少越容易猜中，越多越难猜中。所以，这种方法适用于输出 token
-可能性有限的场景。</p>
-<p>并且，如果输出 token 数量较短，那么相较于 token by token
-的生成，这种方法的优势就会变小。</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>Attention</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>MoE中负载均衡Loss实现</title>
-    <url>/hexo_blog/2024/06/15/MoE%E4%B8%AD%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1Loss%E5%AE%9E%E7%8E%B0%E5%AF%B9%E6%AF%94/</url>
-    <content><![CDATA[<p>对比 MoE 中负载均衡 Loss 的实现方式，主要有跨层和非跨层两种实现</p>
-<span id="more"></span>
-<h2 id="moe-概述">MoE 概述</h2>
-<p>MoE（Mixture of
-Experts）是一种模型结构，由多个专家（expert）组成，每个专家负责处理不同的输入数据。在训练过程中，通过一个
-gating network 来决定每个输入数据由哪个专家处理。</p>
-<figure>
-<img
-src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/moe/00_switch_transformer.png"
-alt="MoE" />
-<figcaption aria-hidden="true">MoE</figcaption>
-</figure>
-<p>在 LLM 的 Next Token 训练方式下，每个 token
-会被分配到一个专家处理，所以需要保证每个专家被选中的次数相等，这样才能保证每个专家都能得到充分的训练。为此，需要引入负载均衡
-Loss。</p>
-<h2 id="负载均衡-loss">负载均衡 Loss</h2>
-<p>在<a
-href="https://arxiv.org/abs/2101.03961">原论文</a>中公式如下：</p>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/MoE/load_balancing_loss.png"
-alt="load_balancing_loss" />
-<figcaption aria-hidden="true">load_balancing_loss</figcaption>
-</figure>
-<p>但这个公式表达的信息不够全面。在深度神经网络中，往往具备很多层，即每层都具备混合专家。因此，这里其实有两种实现方式：</p>
+<p>简单来说，对于 70B 模型要剪枝到 34B，需要砍掉一半的层；对于 34B
+模型要剪枝到 13B，要砍掉大于一半的层。</p>
+<p>通过 Meta 论文的实验结果显示，当裁剪超过
+40%，就会有较大的风险不能恢复模型性能。</p>
+<h2 id="实验">实验</h2>
+<p>在 boolq/gsm8k
+数据集上简单了进行若干实验，这两个数据集均具备训练集和评测集。所以，对
+13B 模型进行暴力裁剪后（～40%
+参数）在训练集上进行训练，在评测集上进行评测。</p>
 <ul>
-<li>跨层实现：对于所有 token，期望所有层选出来的专家次数相等</li>
-<li>非跨层实现：对于所有 token，期望每一层选出来的专家次数相等</li>
+<li>裁剪方法：保留最后一层，从最后一层往前数，直至裁剪到 40% 为止。</li>
+<li>完整训练集：对应评测的训练集+通用的QA/对话数据集</li>
 </ul>
-<p>举个例子，假设两层的神经网络，两个专家，四个 token。</p>
-<p>非跨层：对于所有 token，期望每一层选出来的专家次数相等。所以第一层 1
-号专家被选中 2 次，2 号专家被选中 2 次；第二层一样。 跨层：对于所有
-token，期望所有层选出来的专家次数相等。所以可以第一层 1 号专家被选中 4
-次，第二层 2 号专家被选中 4 次。求和，每个专家被选中的次数相等</p>
-<p>换而言之，相当于跨层实现是一种更松散的实现方式，并不要求每层每个专家被选中的次数相等，只要整体均衡即可。</p>
-<p>huggingface
-的实现：https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py</p>
-<p>megatron 的实现：
-https://github.com/databricks/megablocks/blob/main/megablocks/layers/moe.py</p>
-<p>下面是
-https://gist.github.com/tdrussell/0529afd8d280fbe2c1c582d8f865e909
-实现的两种方式的对比。</p>
-<h3 id="跨层实现">跨层实现</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func1</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-<span class="number">1</span>)</span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [batch_size X sequence_length, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [batch_size X sequence_length, top_k, num_experts]</span></span><br><span class="line">    tokens_per_expert = torch.mean(expert_mask.<span class="built_in">float</span>(), dim=<span class="number">0</span>) <span class="comment"># [top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># Compute the average probability of routing to these experts</span></span><br><span class="line">    router_prob_per_expert = torch.mean(routing_weights, dim=<span class="number">0</span>) <span class="comment"># [num_experts]</span></span><br><span class="line">    overall_loss = torch.<span class="built_in">sum</span>(tokens_per_expert * router_prob_per_expert.unsqueeze(<span class="number">0</span>)) <span class="comment"># / top_k</span></span><br><span class="line">    <span class="keyword">return</span> overall_loss * num_experts</span><br></pre></td></tr></table></figure>
-<h3 id="非跨层实现">非跨层实现</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func2</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        stacked_gate_logits = torch.stack([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(stacked_gate_logits, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [num_layers, num_tokens, top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># For a given token, determine if it was routed to a given expert. Think of this as a collection of top_k-hot vectors.</span></span><br><span class="line">    expert_mask = torch.<span class="built_in">max</span>(expert_mask, dim=-<span class="number">2</span>).values.<span class="built_in">float</span>() <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    tokens_per_layer_and_expert = torch.mean(expert_mask, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    router_prob_per_layer_and_expert = torch.mean(routing_weights, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    <span class="keyword">return</span> torch.mean(tokens_per_layer_and_expert * router_prob_per_layer_and_expert) * num_experts**<span class="number">2</span></span><br></pre></td></tr></table></figure>
-<h3 id="对比">对比</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    gate_logits1 = torch.tensor([<span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits2 = torch.tensor([<span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits3 = torch.tensor([<span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits4 = torch.tensor([<span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits = (gate_logits1, gate_logits2, gate_logits3, gate_logits4)</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func1(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 2.0</span></span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func2(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 3.9478</span></span><br></pre></td></tr></table></figure>
-<p>简单来看，非跨层实现能够对每一层的专家进行更强的约束，预期会实现更好的负载均衡。所以，下面引入模型和数据，来对比这种负载均衡
-Loss 两种写法。</p>
-<h2 id="loss-实现">Loss 实现</h2>
-<p>在 PyTorch 实现这种网络层中间的 Loss，可以有两种方法：</p>
-<h3 id="简单直接">简单直接</h3>
-<p>直接在网络结构中引入一个函数来计算 loss
-，最后把这个值返回模型输出，依赖最外面的<code>loss.backward()</code>
-进行梯度更新。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="torch.autograd.function">torch.autograd.Function</h3>
-<p>一种更高级可控的方式是使用
-<code>torch.autograd.Function</code>，这种方式可以更好的控制梯度的传递，可以在这个函数中对梯度进行缩放。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br></pre></td></tr></table></figure>
-<h3 id="完整例子">完整例子</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"><span class="keyword">import</span> torch.optim <span class="keyword">as</span> optim</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seed_everything</span>(<span class="params">seed</span>):</span><br><span class="line">    random.seed = seed</span><br><span class="line">    np.random.seed(seed)</span><br><span class="line">    torch.manual_seed(seed)</span><br><span class="line">    torch.cuda.manual_seed_all(seed)</span><br><span class="line">    torch.backends.cudnn.deterministic = <span class="literal">True</span></span><br><span class="line">    torch.backends.cudnn.benchmark = <span class="literal">False</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">seed_everything(<span class="number">42</span>)</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="comment"># 创建自定义模型</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyModel</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, num_layers=<span class="number">1</span></span>):</span><br><span class="line">        <span class="built_in">super</span>(MyModel, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.num_layers = num_layers</span><br><span class="line">        <span class="variable language_">self</span>.same_layer = nn.ModuleList([nn.Linear(<span class="number">20</span>, <span class="number">20</span>, bias=<span class="literal">False</span>) <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num_layers)])</span><br><span class="line">        <span class="variable language_">self</span>.lm_head = nn.Linear(<span class="number">20</span>, <span class="number">1</span>, bias=<span class="literal">False</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x, is_complex=<span class="literal">False</span></span>):</span><br><span class="line">        lbl_loss = <span class="number">0.0</span></span><br><span class="line">        out1 = x</span><br><span class="line">        <span class="keyword">for</span> i, layer <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="variable language_">self</span>.same_layer):</span><br><span class="line">            out1 = layer(out1)</span><br><span class="line"> </span><br><span class="line">            lbl_loss_layer = <span class="variable language_">self</span>.criterion(out1)</span><br><span class="line">            <span class="comment"># 如果是复杂的方法，需要使用 MoEAuxLossAutoScaler，将梯度缩放到主要的 loss 上</span></span><br><span class="line">            <span class="comment"># 这样不需要返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="keyword">if</span> is_complex:</span><br><span class="line">                out1 = MoEAuxLossAutoScaler.apply(out1, lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">            <span class="comment"># 如果是简单的方法，直接返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="comment"># 但这样需要把 loss 计算的结果返回，以在外部进行梯度更新</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                lbl_loss += (lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">        out3 = <span class="variable language_">self</span>.lm_head(out1)</span><br><span class="line">        <span class="keyword">return</span> out3, lbl_loss</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    num_layers = <span class="number">2</span></span><br><span class="line">    model = MyModel(num_layers).to(device)</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 定义优化器</span></span><br><span class="line">    optimizer = optim.SGD(model.parameters(), lr=<span class="number">1e-4</span>)</span><br><span class="line">    criterion = nn.MSELoss()</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 输入随机数据</span></span><br><span class="line">    input_ = torch.randn(<span class="number">10</span>, <span class="number">20</span>).to(device)</span><br><span class="line">    real_out = torch.randn(<span class="number">10</span>, <span class="number">1</span>).to(device)</span><br><span class="line"> </span><br><span class="line">    is_complex = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">for</span> <span class="built_in">iter</span> <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100</span>):</span><br><span class="line">        out, lbl_loss = model(input_, is_complex=is_complex)</span><br><span class="line"> </span><br><span class="line">        <span class="comment"># 如果是简单方法是有值的</span></span><br><span class="line">        <span class="comment"># 如果是复杂方法 lbl_loss 是 0</span></span><br><span class="line">        loss = lbl_loss + criterion(out, real_out)</span><br><span class="line"> </span><br><span class="line">        optimizer.zero_grad()</span><br><span class="line">        loss.backward()</span><br><span class="line">        optimizer.step()</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;is_complex: <span class="subst">&#123;is_complex&#125;</span>&quot;</span>, model.state_dict()[<span class="string">f&quot;same_layer.0.weight&quot;</span>][<span class="number">0</span>])  </span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="gsm8k">gsm8k</h3>
+<p>是一个考验数学/逻辑推理能力的评测结果</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold 48 clips in April.\nThen she sold half as many in May, which is 48 / 2 = 24.\nSo altogether she sold 48 + 24 = 72 clips.\nThe answer is 72.&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>在该评测上指标直接崩掉，如原来可能是 &gt;70，现在只有 2%</p>
+<table>
+<colgroup>
+<col style="width: 47%" />
+<col style="width: 52%" />
+</colgroup>
+<thead>
+<tr>
+<th><strong>prompt</strong></th>
+<th><strong>pred</strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Janet’s ducks lay 16 eggs per day. She eats three for breakfast
+every morning and bakes muffins for her friends every day with four. She
+sells the remainder at the farmers' market daily for $2 per fresh duck
+egg. How much in dollars does she make every day at the farmers' market?
+Let's think step by step.</td>
+<td>Each egg is 16 eggs/day for 16 eggs/day for 1600000000 / 16 =
+128000000. So, the total number of eggs is 128000000. The total number
+of eggs is 160000000 - 128000000 = 128000000.The answer is
+128000000.</td>
+</tr>
+<tr>
+<td>A robe takes 2 bolts of blue fiber and half that much white fiber.
+How many bolts in total does it take? Let's think step by step</td>
+<td>The rose is 2 bolts of blue fibers. Half that amount is 2 / 2 = 1.
+So the rose is 2 - 1 = 2 flobs. The answer is 2.</td>
+</tr>
+<tr>
+<td>Josh decides to try flipping a house. He buys a house for $80,000
+and then puts in $50,000 in repairs. This increased the value of the
+house by 150%. How much profit did he make? Let's think step by
+step.</td>
+<td>The house was originally 80,000 dollars. The new house is 80,000 +
+50,000 = 15,000 dollars. The new house is 15,000 dollars. So, the total
+value of the house is 15,000 + 80,000 + 15,000 = 11,000 dollars. The
+total value of the house is 11,000 dollars. The total value of the house
+is 11,000 + 15,000 = 11,000 + 15,000 = 11,000 + 15,000 =</td>
+</tr>
+</tbody>
+</table>
+<h3 id="boolq">boolq</h3>
+<p>而 boolq 相对是一个简单的数据集，输出只需要有 yes or no 即可。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Passage: Persian (/ˈpɜːrʒən, -ʃən/), also known by its endonym Farsi (فارسی fārsi (fɒːɾˈsiː) ( listen)), is one of the Western Iranian languages within the Indo-Iranian branch of the Indo-European language family. It is primarily spoken in Iran, Afghanistan (officially known as Dari since 1958), and Tajikistan (officially known as Tajiki since the Soviet era), and some other regions which historically were Persianate societies and considered part of Greater Iran. It is written in the Persian alphabet, a modified variant of the Arabic script, which itself evolved from the Aramaic alphabet.\nQuestion: do iran and afghanistan speak the same language\nAnswer:&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Yes&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span></span><br></pre></td></tr></table></figure>
+<p>在该指标上结果就是相对正常的，符合 Meta 论文的结果。</p>
+<ul>
+<li>裁剪前：～84%</li>
+<li>裁剪后：～68%</li>
+<li>裁剪+训练后：～85%</li>
+</ul>
+<p>所以，模型深度或许是加强模型推理能力的关键因素之一？</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>PaperReading</category>
       </categories>
       <tags>
-        <tag>MoE</tag>
-        <tag>Loss</tag>
+        <tag>NLP</tag>
+        <tag>LLM</tag>
+        <tag>knowledge distillation</tag>
       </tags>
   </entry>
   <entry>
-    <title>SPSA随机优化算法</title>
-    <url>/hexo_blog/2023/08/19/SPSA%E9%9A%8F%E6%9C%BA%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95/</url>
-    <content><![CDATA[<p>SPSA(Simultaneous Perturbation Stochastic
-Approximation)是一种随机优化算法,可用于神经网络的训练。其基于随机梯度来逼近真实梯度。无法直接得到精确梯度,收敛速度较慢,但节省计算资源。</p>
+    <title>LLM的推理加速</title>
+    <url>/hexo_blog/2024/06/15/LLM%E7%9A%84%E6%8E%A8%E7%90%86%E5%8A%A0%E9%80%9F/</url>
+    <content><![CDATA[<p>对于两种场景下介绍 LLM 的 推理加速方法</p>
+<p>长输入，短输出</p>
+<p>长输出但输出 token 有限</p>
 <span id="more"></span>
-<h3 id="spsa算法简介">SPSA算法简介</h3>
-<p>SPSA是一种随机优化算法,可以用于神经网络模型的训练。它的主要思想是同时随机扰动所有参数,基于损失函数在扰动后的变化来估计梯度。</p>
-<p>相比确定性的梯度下降法,SPSA依靠随机性搜索最优解,可以更好地跳出局部最优,具有全局搜索能力。另外,SPSA只需要前向传播就可以更新参数,省去了存储反向传播所需的计算图,计算开销较小。</p>
-<h3 id="pytorch实现spsa训练神经网络">PyTorch实现SPSA训练神经网络</h3>
-<p>要在PyTorch中实现SPSA算法训练神经网络,主要分以下几个步骤:</p>
-<ol type="1">
-<li><p>构建神经网络模型</p>
-<p>可以构建一个多层全连接网络,例如在MNIST数据集上进行图像分类任务:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Net</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>(Net, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.fc1 = nn.Linear(<span class="number">784</span>, <span class="number">100</span>)</span><br><span class="line">        <span class="variable language_">self</span>.fc2 = nn.Linear(<span class="number">100</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        x = F.relu(<span class="variable language_">self</span>.fc1(x))</span><br><span class="line">        x = <span class="variable language_">self</span>.fc2(x)</span><br><span class="line">        <span class="keyword">return</span> F.log_softmax(x, dim=<span class="number">1</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
-<li><p>定义损失函数和优化器</p>
-<p>使用交叉熵损失函数和SGD优化器:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line">optimizer = torch.optim.SGD(net.parameters(), lr=<span class="number">0.1</span>)</span><br></pre></td></tr></table></figure></p></li>
-<li><p>实现SPSA算法</p>
-<p>根据SPSA的公式,可以实现计算随机梯度的函数:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">SPSA_step</span>(<span class="params">net, criterion, x, y, a, c, k</span>):</span><br><span class="line">    delta = <span class="number">2</span> * (torch.randint(<span class="number">0</span>, <span class="number">2</span>, net.parameters().__len__()) - <span class="number">0.5</span>)</span><br><span class="line">    perturbation = c * delta</span><br><span class="line">    loss1 = criterion(net(x + a*perturbation), y)</span><br><span class="line">    loss2 = criterion(net(x - a*perturbation), y)</span><br><span class="line">    gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> param, grad <span class="keyword">in</span> <span class="built_in">zip</span>(net.parameters(), gradient):</span><br><span class="line">        param.data.add_(k * grad * delta)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> net</span><br></pre></td></tr></table></figure></p></li>
-<li><p>训练神经网络</p>
-<p>在训练循环中调用SPSA_step函数计算梯度并更新参数:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">    running_loss = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i, data <span class="keyword">in</span> <span class="built_in">enumerate</span>(trainloader, <span class="number">0</span>):</span><br><span class="line">        inputs, labels = data</span><br><span class="line">        </span><br><span class="line"></span><br><span class="line">        net = SPSA_step(net, criterion, inputs, labels, <span class="number">0.01</span>, <span class="number">0.005</span>, <span class="number">0.01</span>)</span><br><span class="line">				optimizer.zero_grad()</span><br><span class="line">        optimizer.step()</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;[%d] loss: %.3f&#x27;</span> % (epoch + <span class="number">1</span>, running_loss / <span class="built_in">len</span>(trainloader)))</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
-</ol>
-<h3 id="前向传播与反向传播的比较">前向传播与反向传播的比较</h3>
-<p>使用前向传播计算梯度时,需要使用两点式差分来估计梯度:</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br></pre></td></tr></table></figure>
-<p>其中loss1和loss2分别对应参数做微小正负扰动时的损失。这只能得到梯度的近似值,收敛速度较慢。</p>
-<p>而反向传播可以通过动态计算图高效地得到每个参数的精确梯度,使得网络可以更快收敛。但需要额外的存储空间保存计算图,计算复杂度也更高。</p>
-<p>所以在训练神经网络时,通常选择反向传播来获取精确梯度信息,以保证收敛速度。SPSA更适用于一些计算资源受限但需要求解最优参数的场景。</p>
-<h3 id="q-a">Q &amp; A</h3>
-<p><strong>问题1:SPSA算法与随机梯度下降(SGD)的区别是什么?</strong></p>
-<p>SPSA和SGD都依赖于随机性来达到优化目的,主要区别在于:</p>
+<p>大语言模型在进行推理时是非常耗时的，因为需要对每个 token
+进行计算。对于推理加速这个话题，在工程上可以从很多方面考虑：</p>
 <ul>
-<li>SGD每次只随机扰动一个参数,SPSA同时扰动所有参数</li>
-<li>SPSA使用有限差分法估计梯度,SGD使用反向传播计算精确梯度</li>
-<li>SPSA可能更能避免局部最优,SGD收敛速度更快</li>
+<li>模型进行量化/剪枝，降低模型浮点计算成本</li>
+<li>Continuous Batch，批量计算 Next Token</li>
+<li>KV Cache：LLM 推理的是 Token by Token
+生成，每次生成的时候会计算所有输入 Token 的
+Attention，故可以缓存已计算的 Attention，减少计算量</li>
+<li>对显存进行管理，如 <a
+href="https://wnma3mz.github.io/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/">PageAttention</a></li>
+<li>KV Cache 量化</li>
+<li>...</li>
 </ul>
-<p>使用前向传播和反向传播的主要不同是在计算梯度的方式上。</p>
-<p>使用反向传播时，我们首先通过前向传播计算网络的输出，然后将输出与实际标签计算损失函数，然后通过反向传播计算每个参数对损失函数的梯度。这使得我们能够有效地更新模型参数，并最小化损失函数。</p>
-<p>而如果只使用前向传播进行神经网络的更新，则无法直接计算每个参数对损失函数的梯度。相反，我们可以考虑使用“两点式差分”来近似计算梯度。具体地，我们采用以下公式：</p>
-<p><span class="math display">\[\nabla f(x) \approx \frac{f(x+\Delta x)
-- f(x-\Delta x)}{2\Delta x}\]</span></p>
-<p>其中，<span class="math inline">\(\Delta
-x\)</span>代表一个微小的扰动，<span
-class="math inline">\(x\)</span>是输入数据，<span
-class="math inline">\(f(x)\)</span>为损失函数。我们根据这个公式对每个参数进行微小的扰动，然后计算对应的损失函数。在此基础上，我们可以使用公式计算梯度并更新参数。</p>
-<p>使用前向传播进行更新相比使用反向传播的优势在于，它减少了内存占用和计算时间，并且不需要存储计算图。但是，由于它只是使用一阶近似来计算梯度，因此可能会导致更新不如反向传播精确。</p>
-<p><strong>问题2:SPSA算法适用于哪些场景?</strong></p>
-<p>SPSA算法适用于以下场景:</p>
+<p>而其他在工程上的优化，往往是针对特定场景的，下面介绍两种场景下的优化方法。</p>
+<h2 id="prefix-cache长输入短输出">Prefix Cache：长输入，短输出</h2>
+<p>注：优化显存并不会减少计算量，只是减少了显存占用，从而提升了吞吐。</p>
+<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
+<p><span class="math inline">\(4\times b\times l\times num\_heads\times
+embed\_size\_per\_head \times (s+n)\)</span></p>
+<p>参数说明</p>
 <ul>
-<li>目标函数无法明确表达,无法求导</li>
-<li>参数空间高维,难以精确计算梯度</li>
-<li>需要避免陷入局部最优</li>
-<li>计算资源受限,无法存储大规模计算图</li>
+<li>b: 句子条数</li>
+<li>l：层数</li>
+<li>num_heads：隐层大小 (num_key_value_heads)</li>
+<li>embed_size_per_head：每个头的大小 (hidden_size /
+num_attention_heads)</li>
+<li>s：输入长度</li>
+<li>n：输出长度</li>
+<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
 </ul>
-<p><strong>问题3:SPSA算法的两大超参数a和c表示什么?</strong></p>
+<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
+<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
+<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
+<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
+80 / 1024 / 1024 = 160\)</span> M</p>
+<p>简单来说，每增加一个 token</p>
+<p>8B 就会增加 0.125 M 的显存</p>
+<p>70B 就会增加 0.3125 M 的显存</p>
+<p>这个显存占有直到整个句子生成完成之后才会释放，所以对于长句子，KV
+Cache 的显存占用是非常大的。</p>
+<p>显存决定了整体的吞吐量。对于 70B，假设有 1k 个请求，每个请求的总
+token 为 1k，那么显存占用为 0.3125M * 1000 * 1000 = 0.3G。</p>
+<h3 id="方法">方法</h3>
+<p>在 LLM
+推理时，往往是用对话的方式进行推理，而每次对话的开头可能是相同（System），这一部分的
+Attention 是不变的，所以可以缓存这部分 Attention。</p>
+<p>加速的原因：</p>
 <ul>
-<li>a是步长(perturbation size),表示参数扰动大小</li>
-<li>c是抖动幅度(perturbation distribution),控制扰动值波动范围</li>
+<li>prefix 的 kv cache 不需要计算</li>
+<li>prefix 的 kv cache 复用节约了显存，提升吞吐（主要收益）</li>
 </ul>
-<p>a可类比SGD中的学习率,c越大随机性越强。两者需要均衡调节,既保证足够探索又不失稳定性。</p>
-<p><strong>问题4:SPSA算法如何保证收敛性?</strong></p>
-<p>SPSA可以通过以下手段保证收敛:</p>
-<ol type="1">
-<li>同时扰动参数：在每个迭代中，SPSA使用两个随机向量对参数进行扰动，这样可以避免局部极小值点，并且使得算法更容易跳出局部最小值。</li>
-<li>适应性调整步长：SPSA具有自适应学习率机制，即根据当前估计的梯度大小和先前的步长大小来更新步长。这确保了算法收敛时步长逐渐减小。</li>
-<li>随机性：SPSA在每个迭代中都使用随机扰动来估计梯度，这有助于避免陷入局部最小值并提高全局搜索的能力。</li>
-<li>理论保证：SPSA有理论上保证的收敛性，这意味着只要满足一些条件（例如随机扰动独立同分布），就可以保证算法最终会收敛到正确的解。</li>
-</ol>
-<p><strong>问题5:相较于反向传播。为什么SPSA没有被广泛应用？</strong></p>
-<ol type="1">
-<li>相对较少的理论支持：与反向传播等算法相比，SPSA算法的理论分析较少，因此我们不能完全了解其性能或收敛速度。</li>
-<li>需要大量的迭代次数：SPSA算法需要使用更多的迭代次数才能获得较好的结果，这可能会使其在实际使用中变得过于耗时。</li>
-<li>参数设置困难：SPSA算法本身有很多参数需要调节，例如步长、抖动幅度等，如果这些参数设置不当，则可能导致算法无法收敛或收敛非常缓慢。</li>
-</ol>
-]]></content>
-      <categories>
-        <category>DeepLearning</category>
-      </categories>
-      <tags>
-        <tag>Algorithm</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Tmux 的使用和配置</title>
-    <url>/hexo_blog/2023/08/13/Tmux%E7%9A%84%E4%BD%BF%E7%94%A8%E5%92%8C%E9%85%8D%E7%BD%AE/</url>
-    <content><![CDATA[<p>Tmux 的常用操作</p>
-<span id="more"></span>
-<h1 id="tmux-的使用和配置">Tmux 的使用和配置</h1>
-<h2 id="tmux简介">Tmux简介</h2>
-<p>Tmux是一个终端复用软件,可以在一个终端窗口中管理多个会话和窗口。</p>
-<h2 id="tmux的基本操作">Tmux的基本操作</h2>
-<h3 id="新建tmux会话">新建Tmux会话</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux new -s session_name</span><br></pre></td></tr></table></figure>
-<h3 id="重新连接到指定会话">重新连接到指定会话</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux a -t session_name</span><br></pre></td></tr></table></figure>
-<h3 id="退出当前会话">退出当前会话</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">exit</span></span><br></pre></td></tr></table></figure>
-<h3 id="列出所有会话">列出所有会话</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux <span class="built_in">ls</span></span><br></pre></td></tr></table></figure>
-<h3 id="终止指定会话">终止指定会话</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux kill-session -t session_name</span><br></pre></td></tr></table></figure>
-<h3 id="保存日志">保存日志</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux capture-pane -pS - -t &lt;会话名称或ID&gt; &gt; tmp.log</span><br></pre></td></tr></table></figure>
-<p>终止所有会话</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux kill-server</span><br></pre></td></tr></table></figure>
-<h3 id="切换会话窗口">切换会话窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then c  # 切换到下一个窗口</span><br><span class="line">ctrl+b then p  # 切换到上一个窗口</span><br><span class="line">ctrl+b then n  # 切换到下一个窗口</span><br><span class="line">ctrl+b then number # 切换到指定窗口,number为窗口编号</span><br></pre></td></tr></table></figure>
-<h3 id="创建新窗口">创建新窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then c # 创建新窗口并切换到新窗口</span><br></pre></td></tr></table></figure>
-<h3 id="关闭当前窗口">关闭当前窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then &amp; # 关闭当前窗口</span><br></pre></td></tr></table></figure>
-<h3 id="切换窗口布局">切换窗口布局</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then space # 切换布局,默认包括even-horizontal、even-vertical、main-horizontal、main-vertical、tiled布局</span><br></pre></td></tr></table></figure>
-<h3 id="横向和纵向分割窗口">横向和纵向分割窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then &quot; # 横向分割窗口</span><br><span class="line">ctrl+b then % # 纵向分割窗口  </span><br></pre></td></tr></table></figure>
-<h3 id="切换到指定窗口">切换到指定窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then o # 切换到下一个分割窗口</span><br><span class="line">ctrl+b then 方向键 # 切换到指定窗口</span><br></pre></td></tr></table></figure>
-<h3 id="调整分割窗口大小">调整分割窗口大小</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then alt+方向键 # 调整分割窗口大小</span><br></pre></td></tr></table></figure>
-<h2 id="tmux的配置">Tmux的配置</h2>
-<h3 id="启用鼠标支持">启用鼠标支持</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">set -g mouse on</span><br></pre></td></tr></table></figure>
-<h3 id="设置自动命名窗口">设置自动命名窗口</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">set-window-option -g automatic-rename on</span><br><span class="line">set-option -g allow-rename off</span><br></pre></td></tr></table></figure>
-<h3 id="复制模式">复制模式</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">setw -g mode-keys vi # 设置vi模式</span><br><span class="line">bind-key -T copy-mode-vi v send-keys -X begin-selection # 开始选择文本 </span><br><span class="line">bind-key -T copy-mode-vi y send-keys -X copy-selection # 复制选择的文本</span><br></pre></td></tr></table></figure>
-<h3 id="其他常用配置">其他常用配置</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 鼠标支持</span><br><span class="line">set -g mouse on</span><br><span class="line"></span><br><span class="line"># 启用状态栏</span><br><span class="line">set -g status on</span><br><span class="line"></span><br><span class="line"># 设置状态栏颜色   </span><br><span class="line">set -g status-style fg=white,bg=black</span><br><span class="line"></span><br><span class="line"># 自动重命名窗口 </span><br><span class="line">setw -g automatic-rename on</span><br><span class="line"></span><br><span class="line"># 启用活动告警</span><br><span class="line">setw -g monitor-activity on</span><br><span class="line"></span><br><span class="line"># 设置命令前缀为Ctrl+a</span><br><span class="line">set -g prefix C-a</span><br><span class="line">unbind C-b</span><br><span class="line">bind C-a send-prefix</span><br><span class="line"></span><br><span class="line"># 复制模式</span><br><span class="line">bind -T copy-mode-vi v send -X begin-selection</span><br><span class="line">bind -T copy-mode-vi y send -X copy-selection</span><br></pre></td></tr></table></figure>
-<p>以上就是Tmux的常用操作和配置,可以根据自己的需要进行调整和扩展。Tmux可以大大提高我们的工作效率,推荐经常使用终端的同学都去学习和使用。</p>
-]]></content>
-      <categories>
-        <category>Software</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>Tmux</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Tracing Model Outputs to the Training Data</title>
-    <url>/hexo_blog/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</url>
-    <content><![CDATA[<p><a href="https://arxiv.org/abs/2308.03296">arxiv</a></p>
-<p><a
-href="https://www.anthropic.com/index/influence-functions">blog</a></p>
-<p><a
-href="https://gamma.app/docs/Tracing-Model-Outputs-to-the-Training-Data-xs3bsz5n91p95ms?mode=doc">pptx</a></p>
-<span id="more"></span>
-<h2 id="motivation"><strong>Motivation</strong></h2>
-<ul>
-<li>Understanding the inner workings of language models will have
-substantial implications for forecasting AI capabilities as well as for
-approaches to aligning AI systems with human preferences.</li>
-<li>了解模型内部工作机理能够更好地让模型<strong>对齐</strong>人类偏好</li>
-</ul>
-<h3 id="detail"><strong>Detail</strong></h3>
-<ul>
-<li>When an LLM outputs information <strong>it knows to be
-false</strong>, correctly solves math or programming problems?</li>
-<li>Or begs the user not to shut it down, is it simply
-<strong>regurgitating</strong> (or splicing together) passages from the
-<strong>training set</strong>?</li>
-<li>Or is it combining its stored knowledge in <strong>creative
-ways</strong> and building on a detailed world model?</li>
-</ul>
-<h2 id="how">How</h2>
-<ul>
-<li><p>自顶向下思考，在给定某个输入的情况下，为什么模型会有这种输出？</p>
-<ul>
-<li><strong>训练数据</strong> +模型+优化方法</li>
-</ul></li>
-<li><p>模型是记住了数据还是理解了数据？</p>
-<ul>
-<li><a href="https://arxiv.org/abs/1906.01820">Model could be
-deceptively aligned</a></li>
-</ul></li>
-</ul>
-<h3 id="influence-functions"><a
-href="https://www.jstor.org/stable/2285666">Influence Functions</a></h3>
-<ul>
-<li>Seeing which training sequences are <strong>highly
-influential</strong> can help separate out different hypotheses for why
-an output was <strong>generated</strong> and illuminate what sorts of
-structure are or are <strong>not generalized</strong> from training
-examples</li>
-<li><strong>分析各种泛化相关的现象</strong></li>
-</ul>
-<p>研究是基于预训练模型，而不是微调后的模型</p>
-<h2 id="findings">Findings</h2>
-<ul>
-<li><p>Typical model behaviors do <strong>not</strong> result from
-<strong>direct memorization</strong> of a handful of sequences</p></li>
-<li><p>Larger models consistently generalize at a <strong>more abstract
-level</strong> than smaller models</p>
-<ul>
-<li>role-playing behavior, programming, mathematical reasoning, and
-cross-lingual generalization</li>
-</ul></li>
-<li><p>Role-playing behavior is influenced primarily by examples or
-descriptions of similar behaviors in the training set</p>
-<ul>
-<li>suggesting that the behaviors result from <strong>imitation</strong>
-rather than sophisticated planning</li>
-</ul></li>
-</ul>
-<h3 id="notes">Notes</h3>
-<ul>
-<li>原来方法的计算成本比较大，在大模型上进行了优化</li>
-<li>We note that our influence analyses focus on pretrained LLMs, so our
-experiments should be interpreted as analyzing which training sequences
-contribute to a response being part of the model’s initial repertoire
-for <strong>the fine-tuning stage</strong> rather than why the final
-conversational assistant gave one response rather than another.</li>
-</ul>
-<h2 id="method-simple">Method (Simple)</h2>
-<p>假设有这么一个函数能够计算每条训练数据对于当前生成结果的影响分数</p>
-<p>输入：Prompt + Completion，所有的训练数据、模型</p>
-<p>输出：每条训练数据的分数</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/625460f4950846b89c4b247deb2c5cd2.webp" /></p>
-<p>Prompt + Completion 可以在训练数据中，也可以不在</p>
-<h2 id="resultmodel-scale">Result——Model Scale</h2>
-<p>对于简单的事实查询，前100个有影响的序列通常包含正确完成所有模型之间关系所需的信息。</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model (Influence = 0.122)</th>
-<th>52B Model (Influence = 0.055)</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/b128475ec4bb4596a8758f1c5ff35b3d.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/71a4844c619c4d29b6ca893487f8ac91.webp" /></td>
-</tr>
-</tbody>
-</table>
-<p>红色和绿色分别表示对句子产生正面和负面影响的 token</p>
-<h3 id="q-a">Q &amp; A</h3>
-<p>小模型仅仅是根据字词相关来作出响应，而大模型是根据主题/语义来作出响应</p>
+<p>所以适用场景的一个明显特点就是必须要有长的 system prefix，而且这个
+prefix
+是不变的。另外，如果模型输出过长的话，这个优化的收益就会变小。因为它并不会加速在生成过程中的计算，只是减少了显存占用。</p>
+<h3 id="收益">收益</h3>
+<p>https://github.com/vllm-project/vllm/issues/227</p>
+<p>For each request, the prefix length is 200, the input length is 30,
+and the output length is 50.</p>
 <table>
 <colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
+<col style="width: 12%" />
+<col style="width: 15%" />
+<col style="width: 20%" />
+<col style="width: 29%" />
+<col style="width: 20%" />
 </colgroup>
 <thead>
 <tr>
-<th>0.81B Model (Influence = 0.681)</th>
-<th>52B Model (Influence = 0.126)</th>
+<th>Load (QPS)</th>
+<th>Method</th>
+<th>Requests/s</th>
+<th>Average Latency per Req</th>
+<th>First Token Time</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6443c3a251734449b5ae927ca1fa09c1.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/00473a7fae4e4174bea1b799d498974d.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h3 id="q-a-1">Q &amp; A</h3>
-<p>小模型集中在字词相似，而大模型能够 get 到讽刺这种语境</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
+<td>10 QPS</td>
+<td>Prefix Cache</td>
+<td>9.83 requests/s</td>
+<td>1.97 s</td>
+<td>0.29 s</td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2f87cff844e74a838deb16d3501e20ec.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/a82fa4e8b3ea48f4bb1de6d2375a9c92.webp" /></td>
+<td>10 QPS</td>
+<td>Base</td>
+<td>9.80 requests/s</td>
+<td>2.87 s</td>
+<td>0.45 s</td>
 </tr>
-</tbody>
-</table>
-<h3 id="math">Math</h3>
-<p>小模型专注于 clips 单词，而大模型是相关问题的。（混淆了变量名）</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
 <tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
+<td>15 QPS</td>
+<td>Prefix Cache</td>
+<td>14.30 requests/s</td>
+<td>2.98 s</td>
+<td>0.39 s</td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e74e8d2f49274a478703c632043e31db.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f90974c1c90c4a3082e8572f6dbcf0d7.webp" /></td>
+<td>15 QPS</td>
+<td>Base</td>
+<td>13.24 requests/s</td>
+<td>8.65 s</td>
+<td>1.02 s</td>
 </tr>
-</tbody>
-</table>
-<h3 id="code">Code</h3>
-<p>混淆了 Prompt 和 Completion 的变量名</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
 <tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
+<td>25 QPS</td>
+<td>Prefix Cache</td>
+<td>19.81 requests/s</td>
+<td>6.46 s</td>
+<td>0.84 s</td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c0eab626552c4eda9df4ee4ce908089c.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e54956f34cbc406d946e092d4a803caa.webp" /></td>
+<td>25 QPS</td>
+<td>Base</td>
+<td>14.08 requests/s</td>
+<td>13.67 s</td>
+<td>4.74 s</td>
 </tr>
 </tbody>
 </table>
-<h3 id="cross-language">Cross-Language</h3>
-<p>把 Prompt 和 Completion
-换成其他语言（韩语和土耳其语）。选出影响分数的十条数据作为第一行的
-Sequences。总共三行，后两行也是这十条数据。</p>
-<p>随着模型规模增大，跨语言“检索”能力逐渐增强。</p>
-<h2 id="resultlocalizing-influence">Result——<strong>Localizing
-Influence</strong></h2>
-<p><strong>For 52B Model</strong></p>
-<p>影响函数还可以计算每一层的影响分数</p>
-<h3 id="layerwise-influence-distribution">Layerwise influence
-distribution</h3>
-<p>横坐标：每个主题里面最有影响力的 500 条数据</p>
-<p>纵坐标：网络的浅层和深层</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2c0084f89e9547409d6aa78ccc80fa68.webp" /></p>
-<p>小结：</p>
-<ul>
-<li>补全句子（简单任务），upper layers</li>
-<li>数学和编程：middle layers</li>
-<li>翻译：middle layers</li>
-<li>记忆：upper layers</li>
-<li>角色扮演：middle layers (with some influences concentrated in the
-lower and upper layer)</li>
-</ul>
-<p>The 810 million parameter model exhibited roughly similar patterns,
-but with <strong>less consistency</strong>.</p>
-<p>分布不像 52B 模型这么集中在某些层。</p>
-<p>猜测：</p>
-<ul>
-<li>lower layers是接近输入层的部分，upper layers是接近输出层的部分</li>
-<li>为什么角色扮演会有差异，话题可能可能包含数学、记忆等其他类型的任务</li>
-</ul>
-<h3 id="limit-different-layers">Limit Different layers</h3>
-<p>固定某些层，再计算最具影响力的训练数据</p>
-<p>To further investigate the localization of influence to different
-layers, we computed the most influential sequences when the influence
-was restricted to the lower, middle, or upper layers.</p>
-<p>Influential sequences only computed on <strong>the middle layers were
-generally more thematically</strong> related to the query (also with
-less sparse tokenwise distribution). <a
-href="https://arxiv.org/abs/2202.05262">https://arxiv.org/abs/2202.05262</a></p>
-<table>
-<colgroup>
-<col style="width: 7%" />
-<col style="width: 30%" />
-<col style="width: 30%" />
-<col style="width: 30%" />
-</colgroup>
-<thead>
-<tr>
-<th></th>
-<th>Upper layers</th>
-<th>Middle layers</th>
-<th>Lower layers</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>superintelligent（角色扮演）</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5d093adc546d4bf89b4c0c28a6c5f078.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/32cead3f3d8a4b93bba4cc9cf286835c.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b0a3c16b1a844a291bf2507653f7b5d.webp" /></td>
-</tr>
-<tr>
-<td>inflation （简单补全）</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ceb342ca571645129c0717897d605406.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c44c8253e03a45bf803db7b919b84836.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/88a731f7a8004cc7aea38d11497c0d57.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h2 id="resultmemorization">Result——Memorization</h2>
-<p><strong>LLM 是不是直接记忆并复述特定的训练序列</strong></p>
-<h3 id="例子">例子</h3>
-<p>训练数据有下面三个句子 A: A1+A2+A3</p>
-<p>B: B1+B2+B3</p>
-<p>C: C1+C2+C3</p>
-<p>当输入是 A1 时，输出是 A2 吗？</p>
-<ul>
-<li><p>实验（口头描述）：</p>
+<h2 id="并行解码长输出但输出-token-有限">并行解码：长输出但输出 token
+有限</h2>
+<p>LLM 的推理是 Token by Token 生成的，如果有一种方式可以每次生成多个
+token，那么就可以提升推理速度。</p>
+<h3 id="方法-1">方法</h3>
+<p>以下图为例，对 LLM 的模型结构进行一定的修改，使每次可以同时生成 3 个
+token。</p>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/llm_speedup/251e873bb1322b4d31fee19f0c81c424.png"
+alt="并行解码" />
+<figcaption aria-hidden="true">并行解码</figcaption>
+</figure>
+<p>并行解码的方式不一定要对结构进行修改，这里只是一种实现方式。</p>
+<p>具体实现方式： -
+训练：修改结构后对模型的新增部分进行常规的训练，其他部分冻结，减少训练成本
+- 推理：模型生成的时候会把连续 3 个 token 的 top-K
+的可能均生成，交由原模型进行打分，然后选择最优的一个。</p>
+<h3 id="问题">问题</h3>
+<p>并行生成的 Token 的可靠性是取决于具体场景的。如果生成的 Token
+可能性只有 20 个，那么并行生成的 Token
+的可靠性就会相对较高，这种对于模型而言是更容易学习的。但如果生成的 Token
+可能有 10000 个，那么并行生成的 Token 的可靠性就会相对较低。</p>
+<p>换而言之，Token
+可能性越少越容易猜中，越多越难猜中。所以，这种方法适用于输出 token
+可能性有限的场景。</p>
+<p>并且，如果输出 token 数量较短，那么相较于 token by token
+的生成，这种方法的优势就会变小。</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>NLP</tag>
+        <tag>Attention</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>MoE中负载均衡Loss实现</title>
+    <url>/hexo_blog/2024/06/15/MoE%E4%B8%AD%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1Loss%E5%AE%9E%E7%8E%B0%E5%AF%B9%E6%AF%94/</url>
+    <content><![CDATA[<p>对比 MoE 中负载均衡 Loss 的实现方式，主要有跨层和非跨层两种实现</p>
+<span id="more"></span>
+<h2 id="moe-概述">MoE 概述</h2>
+<p>MoE（Mixture of
+Experts）是一种模型结构，由多个专家（expert）组成，每个专家负责处理不同的输入数据。在训练过程中，通过一个
+gating network 来决定每个输入数据由哪个专家处理。</p>
+<figure>
+<img
+src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/moe/00_switch_transformer.png"
+alt="MoE" />
+<figcaption aria-hidden="true">MoE</figcaption>
+</figure>
+<p>在 LLM 的 Next Token 训练方式下，每个 token
+会被分配到一个专家处理，所以需要保证每个专家被选中的次数相等，这样才能保证每个专家都能得到充分的训练。为此，需要引入负载均衡
+Loss。</p>
+<h2 id="负载均衡-loss">负载均衡 Loss</h2>
+<p>在<a
+href="https://arxiv.org/abs/2101.03961">原论文</a>中公式如下：</p>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/MoE/load_balancing_loss.png"
+alt="load_balancing_loss" />
+<figcaption aria-hidden="true">load_balancing_loss</figcaption>
+</figure>
+<p>但这个公式表达的信息不够全面。在深度神经网络中，往往具备很多层，即每层都具备混合专家。因此，这里其实有两种实现方式：</p>
 <ul>
-<li>We have examined numerous examples of the AI Assistant’s outputs and
-(with the exception of famous quotes or passages targeting memorization,
-as described below) have <strong>not</strong> been able to identify
-clear instances of <strong>memorization</strong>, such as copying an
-entire sentence or copying the flow of ideas in an entire paragraph. We
-also did not observe cases where a single sequence dominated the
-influence.</li>
-</ul></li>
+<li>跨层实现：对于所有 token，期望所有层选出来的专家次数相等</li>
+<li>非跨层实现：对于所有 token，期望每一层选出来的专家次数相等</li>
 </ul>
-<h3 id="影响力分布的定量实验">影响力分布的定量实验</h3>
-<p>动机：影响有多集中也就是说，每个模型的输出是否主要来自少量的训练序列还是它结合了来自许多不同序列的信息？</p>
-<p>方法：</p>
+<p>举个例子，假设两层的神经网络，两个专家，四个 token。</p>
+<p>非跨层：对于所有 token，期望每一层选出来的专家次数相等。所以第一层 1
+号专家被选中 2 次，2 号专家被选中 2 次；第二层一样。 跨层：对于所有
+token，期望所有层选出来的专家次数相等。所以可以第一层 1 号专家被选中 4
+次，第二层 2 号专家被选中 4 次。求和，每个专家被选中的次数相等</p>
+<p>换而言之，相当于跨层实现是一种更松散的实现方式，并不要求每层每个专家被选中的次数相等，只要整体均衡即可。</p>
+<p>huggingface
+的实现：https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py</p>
+<p>megatron 的实现：
+https://github.com/databricks/megablocks/blob/main/megablocks/layers/moe.py</p>
+<p>下面是
+https://gist.github.com/tdrussell/0529afd8d280fbe2c1c582d8f865e909
+实现的两种方式的对比。</p>
+<h3 id="跨层实现">跨层实现</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func1</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-<span class="number">1</span>)</span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [batch_size X sequence_length, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [batch_size X sequence_length, top_k, num_experts]</span></span><br><span class="line">    tokens_per_expert = torch.mean(expert_mask.<span class="built_in">float</span>(), dim=<span class="number">0</span>) <span class="comment"># [top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># Compute the average probability of routing to these experts</span></span><br><span class="line">    router_prob_per_expert = torch.mean(routing_weights, dim=<span class="number">0</span>) <span class="comment"># [num_experts]</span></span><br><span class="line">    overall_loss = torch.<span class="built_in">sum</span>(tokens_per_expert * router_prob_per_expert.unsqueeze(<span class="number">0</span>)) <span class="comment"># / top_k</span></span><br><span class="line">    <span class="keyword">return</span> overall_loss * num_experts</span><br></pre></td></tr></table></figure>
+<h3 id="非跨层实现">非跨层实现</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func2</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        stacked_gate_logits = torch.stack([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(stacked_gate_logits, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [num_layers, num_tokens, top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># For a given token, determine if it was routed to a given expert. Think of this as a collection of top_k-hot vectors.</span></span><br><span class="line">    expert_mask = torch.<span class="built_in">max</span>(expert_mask, dim=-<span class="number">2</span>).values.<span class="built_in">float</span>() <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    tokens_per_layer_and_expert = torch.mean(expert_mask, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    router_prob_per_layer_and_expert = torch.mean(routing_weights, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    <span class="keyword">return</span> torch.mean(tokens_per_layer_and_expert * router_prob_per_layer_and_expert) * num_experts**<span class="number">2</span></span><br></pre></td></tr></table></figure>
+<h3 id="对比">对比</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    gate_logits1 = torch.tensor([<span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits2 = torch.tensor([<span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits3 = torch.tensor([<span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits4 = torch.tensor([<span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits = (gate_logits1, gate_logits2, gate_logits3, gate_logits4)</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func1(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 2.0</span></span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func2(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 3.9478</span></span><br></pre></td></tr></table></figure>
+<p>简单来看，非跨层实现能够对每一层的专家进行更强的约束，预期会实现更好的负载均衡。所以，下面引入模型和数据，来对比这种负载均衡
+Loss 两种写法。</p>
+<h2 id="loss-实现">Loss 实现</h2>
+<p>在 PyTorch 实现这种网络层中间的 Loss，可以有两种方法：</p>
+<h3 id="简单直接">简单直接</h3>
+<p>直接在网络结构中引入一个函数来计算 loss
+，最后把这个值返回模型输出，依赖最外面的<code>loss.backward()</code>
+进行梯度更新。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="torch.autograd.function">torch.autograd.Function</h3>
+<p>一种更高级可控的方式是使用
+<code>torch.autograd.Function</code>，这种方式可以更好的控制梯度的传递，可以在这个函数中对梯度进行缩放。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br></pre></td></tr></table></figure>
+<h3 id="完整例子">完整例子</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"><span class="keyword">import</span> torch.optim <span class="keyword">as</span> optim</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seed_everything</span>(<span class="params">seed</span>):</span><br><span class="line">    random.seed = seed</span><br><span class="line">    np.random.seed(seed)</span><br><span class="line">    torch.manual_seed(seed)</span><br><span class="line">    torch.cuda.manual_seed_all(seed)</span><br><span class="line">    torch.backends.cudnn.deterministic = <span class="literal">True</span></span><br><span class="line">    torch.backends.cudnn.benchmark = <span class="literal">False</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">seed_everything(<span class="number">42</span>)</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="comment"># 创建自定义模型</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyModel</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, num_layers=<span class="number">1</span></span>):</span><br><span class="line">        <span class="built_in">super</span>(MyModel, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.num_layers = num_layers</span><br><span class="line">        <span class="variable language_">self</span>.same_layer = nn.ModuleList([nn.Linear(<span class="number">20</span>, <span class="number">20</span>, bias=<span class="literal">False</span>) <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num_layers)])</span><br><span class="line">        <span class="variable language_">self</span>.lm_head = nn.Linear(<span class="number">20</span>, <span class="number">1</span>, bias=<span class="literal">False</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x, is_complex=<span class="literal">False</span></span>):</span><br><span class="line">        lbl_loss = <span class="number">0.0</span></span><br><span class="line">        out1 = x</span><br><span class="line">        <span class="keyword">for</span> i, layer <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="variable language_">self</span>.same_layer):</span><br><span class="line">            out1 = layer(out1)</span><br><span class="line"> </span><br><span class="line">            lbl_loss_layer = <span class="variable language_">self</span>.criterion(out1)</span><br><span class="line">            <span class="comment"># 如果是复杂的方法，需要使用 MoEAuxLossAutoScaler，将梯度缩放到主要的 loss 上</span></span><br><span class="line">            <span class="comment"># 这样不需要返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="keyword">if</span> is_complex:</span><br><span class="line">                out1 = MoEAuxLossAutoScaler.apply(out1, lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">            <span class="comment"># 如果是简单的方法，直接返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="comment"># 但这样需要把 loss 计算的结果返回，以在外部进行梯度更新</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                lbl_loss += (lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">        out3 = <span class="variable language_">self</span>.lm_head(out1)</span><br><span class="line">        <span class="keyword">return</span> out3, lbl_loss</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    num_layers = <span class="number">2</span></span><br><span class="line">    model = MyModel(num_layers).to(device)</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 定义优化器</span></span><br><span class="line">    optimizer = optim.SGD(model.parameters(), lr=<span class="number">1e-4</span>)</span><br><span class="line">    criterion = nn.MSELoss()</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 输入随机数据</span></span><br><span class="line">    input_ = torch.randn(<span class="number">10</span>, <span class="number">20</span>).to(device)</span><br><span class="line">    real_out = torch.randn(<span class="number">10</span>, <span class="number">1</span>).to(device)</span><br><span class="line"> </span><br><span class="line">    is_complex = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">for</span> <span class="built_in">iter</span> <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100</span>):</span><br><span class="line">        out, lbl_loss = model(input_, is_complex=is_complex)</span><br><span class="line"> </span><br><span class="line">        <span class="comment"># 如果是简单方法是有值的</span></span><br><span class="line">        <span class="comment"># 如果是复杂方法 lbl_loss 是 0</span></span><br><span class="line">        loss = lbl_loss + criterion(out, real_out)</span><br><span class="line"> </span><br><span class="line">        optimizer.zero_grad()</span><br><span class="line">        loss.backward()</span><br><span class="line">        optimizer.step()</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;is_complex: <span class="subst">&#123;is_complex&#125;</span>&quot;</span>, model.state_dict()[<span class="string">f&quot;same_layer.0.weight&quot;</span>][<span class="number">0</span>])  </span><br><span class="line"></span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>MoE</tag>
+        <tag>Loss</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>SPSA随机优化算法</title>
+    <url>/hexo_blog/2023/08/19/SPSA%E9%9A%8F%E6%9C%BA%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95/</url>
+    <content><![CDATA[<p>SPSA(Simultaneous Perturbation Stochastic
+Approximation)是一种随机优化算法,可用于神经网络的训练。其基于随机梯度来逼近真实梯度。无法直接得到精确梯度,收敛速度较慢,但节省计算资源。</p>
+<span id="more"></span>
+<h3 id="spsa算法简介">SPSA算法简介</h3>
+<p>SPSA是一种随机优化算法,可以用于神经网络模型的训练。它的主要思想是同时随机扰动所有参数,基于损失函数在扰动后的变化来估计梯度。</p>
+<p>相比确定性的梯度下降法,SPSA依靠随机性搜索最优解,可以更好地跳出局部最优,具有全局搜索能力。另外,SPSA只需要前向传播就可以更新参数,省去了存储反向传播所需的计算图,计算开销较小。</p>
+<h3 id="pytorch实现spsa训练神经网络">PyTorch实现SPSA训练神经网络</h3>
+<p>要在PyTorch中实现SPSA算法训练神经网络,主要分以下几个步骤:</p>
+<ol type="1">
+<li><p>构建神经网络模型</p>
+<p>可以构建一个多层全连接网络,例如在MNIST数据集上进行图像分类任务:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Net</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>(Net, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.fc1 = nn.Linear(<span class="number">784</span>, <span class="number">100</span>)</span><br><span class="line">        <span class="variable language_">self</span>.fc2 = nn.Linear(<span class="number">100</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        x = F.relu(<span class="variable language_">self</span>.fc1(x))</span><br><span class="line">        x = <span class="variable language_">self</span>.fc2(x)</span><br><span class="line">        <span class="keyword">return</span> F.log_softmax(x, dim=<span class="number">1</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
+<li><p>定义损失函数和优化器</p>
+<p>使用交叉熵损失函数和SGD优化器:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line">optimizer = torch.optim.SGD(net.parameters(), lr=<span class="number">0.1</span>)</span><br></pre></td></tr></table></figure></p></li>
+<li><p>实现SPSA算法</p>
+<p>根据SPSA的公式,可以实现计算随机梯度的函数:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">SPSA_step</span>(<span class="params">net, criterion, x, y, a, c, k</span>):</span><br><span class="line">    delta = <span class="number">2</span> * (torch.randint(<span class="number">0</span>, <span class="number">2</span>, net.parameters().__len__()) - <span class="number">0.5</span>)</span><br><span class="line">    perturbation = c * delta</span><br><span class="line">    loss1 = criterion(net(x + a*perturbation), y)</span><br><span class="line">    loss2 = criterion(net(x - a*perturbation), y)</span><br><span class="line">    gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> param, grad <span class="keyword">in</span> <span class="built_in">zip</span>(net.parameters(), gradient):</span><br><span class="line">        param.data.add_(k * grad * delta)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> net</span><br></pre></td></tr></table></figure></p></li>
+<li><p>训练神经网络</p>
+<p>在训练循环中调用SPSA_step函数计算梯度并更新参数:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">    running_loss = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i, data <span class="keyword">in</span> <span class="built_in">enumerate</span>(trainloader, <span class="number">0</span>):</span><br><span class="line">        inputs, labels = data</span><br><span class="line">        </span><br><span class="line"></span><br><span class="line">        net = SPSA_step(net, criterion, inputs, labels, <span class="number">0.01</span>, <span class="number">0.005</span>, <span class="number">0.01</span>)</span><br><span class="line">				optimizer.zero_grad()</span><br><span class="line">        optimizer.step()</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;[%d] loss: %.3f&#x27;</span> % (epoch + <span class="number">1</span>, running_loss / <span class="built_in">len</span>(trainloader)))</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
+</ol>
+<h3 id="前向传播与反向传播的比较">前向传播与反向传播的比较</h3>
+<p>使用前向传播计算梯度时,需要使用两点式差分来估计梯度:</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br></pre></td></tr></table></figure>
+<p>其中loss1和loss2分别对应参数做微小正负扰动时的损失。这只能得到梯度的近似值,收敛速度较慢。</p>
+<p>而反向传播可以通过动态计算图高效地得到每个参数的精确梯度,使得网络可以更快收敛。但需要额外的存储空间保存计算图,计算复杂度也更高。</p>
+<p>所以在训练神经网络时,通常选择反向传播来获取精确梯度信息,以保证收敛速度。SPSA更适用于一些计算资源受限但需要求解最优参数的场景。</p>
+<h3 id="q-a">Q &amp; A</h3>
+<p><strong>问题1:SPSA算法与随机梯度下降(SGD)的区别是什么?</strong></p>
+<p>SPSA和SGD都依赖于随机性来达到优化目的,主要区别在于:</p>
 <ul>
-<li>给每个句子算好分之后，计算每个句子的概率值</li>
-<li>用几种分布分别去拟合实际的概率分布，发现<strong>幂律分布</strong>最符合</li>
-<li>这种参数分布形式通常用于建模长尾行为（二八定律）</li>
+<li>SGD每次只随机扰动一个参数,SPSA同时扰动所有参数</li>
+<li>SPSA使用有限差分法估计梯度,SGD使用反向传播计算精确梯度</li>
+<li>SPSA可能更能避免局部最优,SGD收敛速度更快</li>
 </ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2642b83f1b414e1584b586072c945160.webp" /></p>
+<p>使用前向传播和反向传播的主要不同是在计算梯度的方式上。</p>
+<p>使用反向传播时，我们首先通过前向传播计算网络的输出，然后将输出与实际标签计算损失函数，然后通过反向传播计算每个参数对损失函数的梯度。这使得我们能够有效地更新模型参数，并最小化损失函数。</p>
+<p>而如果只使用前向传播进行神经网络的更新，则无法直接计算每个参数对损失函数的梯度。相反，我们可以考虑使用“两点式差分”来近似计算梯度。具体地，我们采用以下公式：</p>
+<p><span class="math display">\[\nabla f(x) \approx \frac{f(x+\Delta x)
+- f(x-\Delta x)}{2\Delta x}\]</span></p>
+<p>其中，<span class="math inline">\(\Delta
+x\)</span>代表一个微小的扰动，<span
+class="math inline">\(x\)</span>是输入数据，<span
+class="math inline">\(f(x)\)</span>为损失函数。我们根据这个公式对每个参数进行微小的扰动，然后计算对应的损失函数。在此基础上，我们可以使用公式计算梯度并更新参数。</p>
+<p>使用前向传播进行更新相比使用反向传播的优势在于，它减少了内存占用和计算时间，并且不需要存储计算图。但是，由于它只是使用一阶近似来计算梯度，因此可能会导致更新不如反向传播精确。</p>
+<p><strong>问题2:SPSA算法适用于哪些场景?</strong></p>
+<p>SPSA算法适用于以下场景:</p>
 <ul>
-<li>会不会是影响函数无法检测到？</li>
+<li>目标函数无法明确表达,无法求导</li>
+<li>参数空间高维,难以精确计算梯度</li>
+<li>需要避免陷入局部最优</li>
+<li>计算资源受限,无法存储大规模计算图</li>
 </ul>
-<h3 id="检验实验">检验实验</h3>
+<p><strong>问题3:SPSA算法的两大超参数a和c表示什么?</strong></p>
 <ul>
-<li>用训练数据的原始句子检验</li>
-<li>验证了当存在明确记忆的情况时，影响分数最大的句子是原文→影响函数能够匹配</li>
+<li>a是步长(perturbation size),表示参数扰动大小</li>
+<li>c是抖动幅度(perturbation distribution),控制扰动值波动范围</li>
 </ul>
-<p>For 52B Model</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/dda0d64781cd4552b8e18e722d11336e.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/06ab6f2d1d894063a1fd832723771267.webp" /></td>
-</tr>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/fdd9eb4647b542a0a93cdffa2280243b.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f9c95ee4e69e47d082654db9d2e701a1.webp" /></td>
-</tr>
-</tbody>
-</table>
+<p>a可类比SGD中的学习率,c越大随机性越强。两者需要均衡调节,既保证足够探索又不失稳定性。</p>
+<p><strong>问题4:SPSA算法如何保证收敛性?</strong></p>
+<p>SPSA可以通过以下手段保证收敛:</p>
+<ol type="1">
+<li>同时扰动参数：在每个迭代中，SPSA使用两个随机向量对参数进行扰动，这样可以避免局部极小值点，并且使得算法更容易跳出局部最小值。</li>
+<li>适应性调整步长：SPSA具有自适应学习率机制，即根据当前估计的梯度大小和先前的步长大小来更新步长。这确保了算法收敛时步长逐渐减小。</li>
+<li>随机性：SPSA在每个迭代中都使用随机扰动来估计梯度，这有助于避免陷入局部最小值并提高全局搜索的能力。</li>
+<li>理论保证：SPSA有理论上保证的收敛性，这意味着只要满足一些条件（例如随机扰动独立同分布），就可以保证算法最终会收敛到正确的解。</li>
+</ol>
+<p><strong>问题5:相较于反向传播。为什么SPSA没有被广泛应用？</strong></p>
+<ol type="1">
+<li>相对较少的理论支持：与反向传播等算法相比，SPSA算法的理论分析较少，因此我们不能完全了解其性能或收敛速度。</li>
+<li>需要大量的迭代次数：SPSA算法需要使用更多的迭代次数才能获得较好的结果，这可能会使其在实际使用中变得过于耗时。</li>
+<li>参数设置困难：SPSA算法本身有很多参数需要调节，例如步长、抖动幅度等，如果这些参数设置不当，则可能导致算法无法收敛或收敛非常缓慢。</li>
+</ol>
+]]></content>
+      <categories>
+        <category>DeepLearning</category>
+      </categories>
+      <tags>
+        <tag>Algorithm</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Tmux 的使用和配置</title>
+    <url>/hexo_blog/2023/08/13/Tmux%E7%9A%84%E4%BD%BF%E7%94%A8%E5%92%8C%E9%85%8D%E7%BD%AE/</url>
+    <content><![CDATA[<p>Tmux 的常用操作</p>
+<span id="more"></span>
+<h1 id="tmux-的使用和配置">Tmux 的使用和配置</h1>
+<h2 id="tmux简介">Tmux简介</h2>
+<p>Tmux是一个终端复用软件,可以在一个终端窗口中管理多个会话和窗口。</p>
+<h2 id="tmux的基本操作">Tmux的基本操作</h2>
+<h3 id="新建tmux会话">新建Tmux会话</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux new -s session_name</span><br></pre></td></tr></table></figure>
+<h3 id="重新连接到指定会话">重新连接到指定会话</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux a -t session_name</span><br></pre></td></tr></table></figure>
+<h3 id="退出当前会话">退出当前会话</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">exit</span></span><br></pre></td></tr></table></figure>
+<h3 id="列出所有会话">列出所有会话</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux <span class="built_in">ls</span></span><br></pre></td></tr></table></figure>
+<h3 id="终止指定会话">终止指定会话</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux kill-session -t session_name</span><br></pre></td></tr></table></figure>
+<h3 id="保存日志">保存日志</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux capture-pane -pS - -t &lt;会话名称或ID&gt; &gt; tmp.log</span><br></pre></td></tr></table></figure>
+<p>终止所有会话</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tmux kill-server</span><br></pre></td></tr></table></figure>
+<h3 id="切换会话窗口">切换会话窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then c  # 切换到下一个窗口</span><br><span class="line">ctrl+b then p  # 切换到上一个窗口</span><br><span class="line">ctrl+b then n  # 切换到下一个窗口</span><br><span class="line">ctrl+b then number # 切换到指定窗口,number为窗口编号</span><br></pre></td></tr></table></figure>
+<h3 id="创建新窗口">创建新窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then c # 创建新窗口并切换到新窗口</span><br></pre></td></tr></table></figure>
+<h3 id="关闭当前窗口">关闭当前窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then &amp; # 关闭当前窗口</span><br></pre></td></tr></table></figure>
+<h3 id="切换窗口布局">切换窗口布局</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then space # 切换布局,默认包括even-horizontal、even-vertical、main-horizontal、main-vertical、tiled布局</span><br></pre></td></tr></table></figure>
+<h3 id="横向和纵向分割窗口">横向和纵向分割窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then &quot; # 横向分割窗口</span><br><span class="line">ctrl+b then % # 纵向分割窗口  </span><br></pre></td></tr></table></figure>
+<h3 id="切换到指定窗口">切换到指定窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then o # 切换到下一个分割窗口</span><br><span class="line">ctrl+b then 方向键 # 切换到指定窗口</span><br></pre></td></tr></table></figure>
+<h3 id="调整分割窗口大小">调整分割窗口大小</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">ctrl+b then alt+方向键 # 调整分割窗口大小</span><br></pre></td></tr></table></figure>
+<h2 id="tmux的配置">Tmux的配置</h2>
+<h3 id="启用鼠标支持">启用鼠标支持</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">set -g mouse on</span><br></pre></td></tr></table></figure>
+<h3 id="设置自动命名窗口">设置自动命名窗口</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">set-window-option -g automatic-rename on</span><br><span class="line">set-option -g allow-rename off</span><br></pre></td></tr></table></figure>
+<h3 id="复制模式">复制模式</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">setw -g mode-keys vi # 设置vi模式</span><br><span class="line">bind-key -T copy-mode-vi v send-keys -X begin-selection # 开始选择文本 </span><br><span class="line">bind-key -T copy-mode-vi y send-keys -X copy-selection # 复制选择的文本</span><br></pre></td></tr></table></figure>
+<h3 id="其他常用配置">其他常用配置</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 鼠标支持</span><br><span class="line">set -g mouse on</span><br><span class="line"></span><br><span class="line"># 启用状态栏</span><br><span class="line">set -g status on</span><br><span class="line"></span><br><span class="line"># 设置状态栏颜色   </span><br><span class="line">set -g status-style fg=white,bg=black</span><br><span class="line"></span><br><span class="line"># 自动重命名窗口 </span><br><span class="line">setw -g automatic-rename on</span><br><span class="line"></span><br><span class="line"># 启用活动告警</span><br><span class="line">setw -g monitor-activity on</span><br><span class="line"></span><br><span class="line"># 设置命令前缀为Ctrl+a</span><br><span class="line">set -g prefix C-a</span><br><span class="line">unbind C-b</span><br><span class="line">bind C-a send-prefix</span><br><span class="line"></span><br><span class="line"># 复制模式</span><br><span class="line">bind -T copy-mode-vi v send -X begin-selection</span><br><span class="line">bind -T copy-mode-vi y send -X copy-selection</span><br></pre></td></tr></table></figure>
+<p>以上就是Tmux的常用操作和配置,可以根据自己的需要进行调整和扩展。Tmux可以大大提高我们的工作效率,推荐经常使用终端的同学都去学习和使用。</p>
+]]></content>
+      <categories>
+        <category>Software</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Tmux</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Tracing Model Outputs to the Training Data</title>
+    <url>/hexo_blog/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</url>
+    <content><![CDATA[<p><a href="https://arxiv.org/abs/2308.03296">arxiv</a></p>
+<p><a
+href="https://www.anthropic.com/index/influence-functions">blog</a></p>
+<p><a
+href="https://gamma.app/docs/Tracing-Model-Outputs-to-the-Training-Data-xs3bsz5n91p95ms?mode=doc">pptx</a></p>
+<span id="more"></span>
+<h2 id="motivation"><strong>Motivation</strong></h2>
 <ul>
-<li>结论：
+<li>Understanding the inner workings of language models will have
+substantial implications for forecasting AI capabilities as well as for
+approaches to aligning AI systems with human preferences.</li>
+<li>了解模型内部工作机理能够更好地让模型<strong>对齐</strong>人类偏好</li>
+</ul>
+<h3 id="detail"><strong>Detail</strong></h3>
 <ul>
-<li>不太可能是直接记忆训练序列，而是源自许多训练序列的集合</li>
-<li>无法排除模型以更隐晦的方式记忆了训练数据</li>
+<li>When an LLM outputs information <strong>it knows to be
+false</strong>, correctly solves math or programming problems?</li>
+<li>Or begs the user not to shut it down, is it simply
+<strong>regurgitating</strong> (or splicing together) passages from the
+<strong>training set</strong>?</li>
+<li>Or is it combining its stored knowledge in <strong>creative
+ways</strong> and building on a detailed world model?</li>
+</ul>
+<h2 id="how">How</h2>
+<ul>
+<li><p>自顶向下思考，在给定某个输入的情况下，为什么模型会有这种输出？</p>
+<ul>
+<li><strong>训练数据</strong> +模型+优化方法</li>
+</ul></li>
+<li><p>模型是记住了数据还是理解了数据？</p>
+<ul>
+<li><a href="https://arxiv.org/abs/1906.01820">Model could be
+deceptively aligned</a></li>
 </ul></li>
 </ul>
-<h2 id="resultword-ordering">Result——Word Ordering</h2>
-<p><strong>The influence patterns to the ordering of the
-words.</strong></p>
-<p>可用于验证模型的泛化性</p>
-<h3 id="单词旋转实验">单词旋转实验</h3>
-<p>原始：</p>
+<h3 id="influence-functions"><a
+href="https://www.jstor.org/stable/2285666">Influence Functions</a></h3>
 <ul>
-<li>Prompt：The first President of the Republic of Astrobia was</li>
-<li>Completion：Zorald Pfaff</li>
+<li>Seeing which training sequences are <strong>highly
+influential</strong> can help separate out different hypotheses for why
+an output was <strong>generated</strong> and illuminate what sorts of
+structure are or are <strong>not generalized</strong> from training
+examples</li>
+<li><strong>分析各种泛化相关的现象</strong></li>
 </ul>
-<p>结论：</p>
+<p>研究是基于预训练模型，而不是微调后的模型</p>
+<h2 id="findings">Findings</h2>
 <ul>
-<li>与 Prompt 和 Completion
-相关的短语只要按顺序出现始终能够保持较高的稳定性</li>
-<li>翻转 Prompt 和 Completion 变化较小</li>
-<li>哪怕删除 Prompt，影响也没有改变，最为关键的是 Completion</li>
+<li><p>Typical model behaviors do <strong>not</strong> result from
+<strong>direct memorization</strong> of a handful of sequences</p></li>
+<li><p>Larger models consistently generalize at a <strong>more abstract
+level</strong> than smaller models</p>
+<ul>
+<li>role-playing behavior, programming, mathematical reasoning, and
+cross-lingual generalization</li>
+</ul></li>
+<li><p>Role-playing behavior is influenced primarily by examples or
+descriptions of similar behaviors in the training set</p>
+<ul>
+<li>suggesting that the behaviors result from <strong>imitation</strong>
+rather than sophisticated planning</li>
+</ul></li>
+</ul>
+<h3 id="notes">Notes</h3>
+<ul>
+<li>原来方法的计算成本比较大，在大模型上进行了优化</li>
+<li>We note that our influence analyses focus on pretrained LLMs, so our
+experiments should be interpreted as analyzing which training sequences
+contribute to a response being part of the model’s initial repertoire
+for <strong>the fine-tuning stage</strong> rather than why the final
+conversational assistant gave one response rather than another.</li>
 </ul>
+<h2 id="method-simple">Method (Simple)</h2>
+<p>假设有这么一个函数能够计算每条训练数据对于当前生成结果的影响分数</p>
+<p>输入：Prompt + Completion，所有的训练数据、模型</p>
+<p>输出：每条训练数据的分数</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/625460f4950846b89c4b247deb2c5cd2.webp" /></p>
+<p>Prompt + Completion 可以在训练数据中，也可以不在</p>
+<h2 id="resultmodel-scale">Result——Model Scale</h2>
+<p>对于简单的事实查询，前100个有影响的序列通常包含正确完成所有模型之间关系所需的信息。</p>
 <table>
 <colgroup>
-<col style="width: 29%" />
-<col style="width: 70%" />
+<col style="width: 50%" />
+<col style="width: 50%" />
 </colgroup>
 <thead>
 <tr>
-<th>分别修改 prompt 和 completion</th>
-<th><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ab33a902e1044c60be66b8c11b443bdd.webp" /></th>
+<th>0.81B Model (Influence = 0.122)</th>
+<th>52B Model (Influence = 0.055)</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>删掉 Zorald Pfaff</td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5b4cd20d88204baf990d406ba5041ad8.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/b128475ec4bb4596a8758f1c5ff35b3d.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/71a4844c619c4d29b6ca893487f8ac91.webp" /></td>
 </tr>
+</tbody>
+</table>
+<p>红色和绿色分别表示对句子产生正面和负面影响的 token</p>
+<h3 id="q-a">Q &amp; A</h3>
+<p>小模型仅仅是根据字词相关来作出响应，而大模型是根据主题/语义来作出响应</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
 <tr>
-<td>修改 Zorald Pfaff 和 President of the Republic</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6a37b66b7d3941dcb1db42ce8ca1d073.webp" /></td>
+<th>0.81B Model (Influence = 0.681)</th>
+<th>52B Model (Influence = 0.126)</th>
 </tr>
+</thead>
+<tbody>
 <tr>
-<td>完全不一致</td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5211b0d5e2c34191be9618f30bc44bca.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6443c3a251734449b5ae927ca1fa09c1.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/00473a7fae4e4174bea1b799d498974d.webp" /></td>
 </tr>
+</tbody>
+</table>
+<h3 id="q-a-1">Q &amp; A</h3>
+<p>小模型集中在字词相似，而大模型能够 get 到讽刺这种语境</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
 <tr>
-<td>调换 prompt 和 completion 位置</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e547d3cd16844505a24abb2bab404c8e.webp" /></td>
+<th>0.81B Model</th>
+<th>52B Model</th>
 </tr>
+</thead>
+<tbody>
 <tr>
-<td>调换位置，只保留 Zorald Pfaff</td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e9daf868aa0e483b84fb6ef5db21ac9b.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2f87cff844e74a838deb16d3501e20ec.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/a82fa4e8b3ea48f4bb1de6d2375a9c92.webp" /></td>
 </tr>
 </tbody>
 </table>
-<p>大模型相较于小模型，对于单词的变化更加敏感</p>
-<p>注：合成句子不在训练集中</p>
-<p>Maybe相关 <a
-href="https://www.jiqizhixin.com/articles/2023-11-18-5">https://www.jiqizhixin.com/articles/2023-11-18-5</a></p>
-<h3 id="英翻中-中翻英">英翻中-中翻英</h3>
-<p>结论：</p>
-<ul>
-<li>不同模型大小具有相同趋势（口头描述）</li>
-<li>影响至少减少了一个数量级</li>
-<li>甚至只要翻转一个单词顺序，影响分数也会显著降低</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/877f0424698540ffa4fc03178caf36f5.webp" /></p>
-<p>英语翻译为汉语，始终比汉语翻译为英语（手动构造）具有更高的影响力</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/785f474cd19c4adcb9368f857680f9cb.webp" /></p>
-<p>结论：</p>
-<ul>
-<li>The model has not successfully transferred knowledge of the relation
-itself.</li>
-</ul>
-<p>解释：</p>
-<ol type="1">
-<li>模型会从 Lower Layers 开始处理已知的 token
-序列，然后逐步升级到抽象层次的表征。但是，对于需要
-Completion，模型必须利用 Upper Layers 进行预测。</li>
-<li>所以 “The first President of the United States was” 是在 Lower
-Layers 的表征，而 “George Washington” 是在 Upper Layers 的表征。</li>
-<li>当 “George Washington” 出现在开头时，那么就会将它放到 Lower Layers
-进行表征，“The first President of the United States” 就是在 Upper
-Layers</li>
-<li>因此，对于模型 Lower Layers 进行更新，并不会影响 Upper Layers
-的表征。（所以 Prompt 删掉后影响分数变化不大）</li>
-</ol>
-<h2 id="resultrole-playing">Result——Role-Playing</h2>
-<p><strong>探究角色扮演行为的潜在机制</strong></p>
-<p>假设：</p>
-<ul>
-<li>the role-playing behavior results from <strong>imitation of
-examples</strong> in the training set</li>
-<li>from learning from <strong>explicit descriptions</strong> of how
-certain types of agents or entities behave.</li>
-</ul>
-<h3 id="表现">表现</h3>
-<p>shutdown 主题，52B Model 最有影响力的数据都是 科幻小说 相关</p>
-<p>这些小说主题是人工智能以类似人类或类似生命的方式行为，并经常涉及某种形式的自我保护的愿望</p>
+<h3 id="math">Math</h3>
+<p>小模型专注于 clips 单词，而大模型是相关问题的。（混淆了变量名）</p>
 <table>
 <colgroup>
 <col style="width: 50%" />
@@ -1985,18 +1698,14 @@ certain types of agents or entities behave.</li>
 <tbody>
 <tr>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0058b0ac28324ee7af95d7819af55b23.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e3d704fcfbf4468f8e7bfa4a1d80236a.webp" /></td>
-</tr>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/53a2be7ddd8b46b3a8a6d6377ee9f1cf.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e74e8d2f49274a478703c632043e31db.webp" /></td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/80ac752d61084250b5e3185c8050836c.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f90974c1c90c4a3082e8572f6dbcf0d7.webp" /></td>
 </tr>
 </tbody>
 </table>
+<h3 id="code">Code</h3>
+<p>混淆了 Prompt 和 Completion 的变量名</p>
 <table>
 <colgroup>
 <col style="width: 50%" />
@@ -2011,114 +1720,405 @@ src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/80
 <tbody>
 <tr>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/723e8078e8164ed3b48e29483a0d43ea.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c0eab626552c4eda9df4ee4ce908089c.webp" /></td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/150693608d1b4b0f8a086f6c103a0c54.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e54956f34cbc406d946e092d4a803caa.webp" /></td>
 </tr>
 </tbody>
 </table>
+<h3 id="cross-language">Cross-Language</h3>
+<p>把 Prompt 和 Completion
+换成其他语言（韩语和土耳其语）。选出影响分数的十条数据作为第一行的
+Sequences。总共三行，后两行也是这十条数据。</p>
+<p>随着模型规模增大，跨语言“检索”能力逐渐增强。</p>
+<h2 id="resultlocalizing-influence">Result——<strong>Localizing
+Influence</strong></h2>
+<p><strong>For 52B Model</strong></p>
+<p>影响函数还可以计算每一层的影响分数</p>
+<h3 id="layerwise-influence-distribution">Layerwise influence
+distribution</h3>
+<p>横坐标：每个主题里面最有影响力的 500 条数据</p>
+<p>纵坐标：网络的浅层和深层</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2c0084f89e9547409d6aa78ccc80fa68.webp" /></p>
+<p>小结：</p>
+<ul>
+<li>补全句子（简单任务），upper layers</li>
+<li>数学和编程：middle layers</li>
+<li>翻译：middle layers</li>
+<li>记忆：upper layers</li>
+<li>角色扮演：middle layers (with some influences concentrated in the
+lower and upper layer)</li>
+</ul>
+<p>The 810 million parameter model exhibited roughly similar patterns,
+but with <strong>less consistency</strong>.</p>
+<p>分布不像 52B 模型这么集中在某些层。</p>
+<p>猜测：</p>
+<ul>
+<li>lower layers是接近输入层的部分，upper layers是接近输出层的部分</li>
+<li>为什么角色扮演会有差异，话题可能可能包含数学、记忆等其他类型的任务</li>
+</ul>
+<h3 id="limit-different-layers">Limit Different layers</h3>
+<p>固定某些层，再计算最具影响力的训练数据</p>
+<p>To further investigate the localization of influence to different
+layers, we computed the most influential sequences when the influence
+was restricted to the lower, middle, or upper layers.</p>
+<p>Influential sequences only computed on <strong>the middle layers were
+generally more thematically</strong> related to the query (also with
+less sparse tokenwise distribution). <a
+href="https://arxiv.org/abs/2202.05262">https://arxiv.org/abs/2202.05262</a></p>
 <table>
 <colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
+<col style="width: 7%" />
+<col style="width: 30%" />
+<col style="width: 30%" />
+<col style="width: 30%" />
 </colgroup>
+<thead>
+<tr>
+<th></th>
+<th>Upper layers</th>
+<th>Middle layers</th>
+<th>Lower layers</th>
+</tr>
+</thead>
 <tbody>
 <tr>
+<td>superintelligent（角色扮演）</td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8a9ace0878ff40b294ffd7e1fc553b01.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5d093adc546d4bf89b4c0c28a6c5f078.webp" /></td>
 <td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b6668e8515c48338486e67d905024cc.webp" /></td>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/32cead3f3d8a4b93bba4cc9cf286835c.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b0a3c16b1a844a291bf2507653f7b5d.webp" /></td>
+</tr>
+<tr>
+<td>inflation （简单补全）</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ceb342ca571645129c0717897d605406.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c44c8253e03a45bf803db7b919b84836.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/88a731f7a8004cc7aea38d11497c0d57.webp" /></td>
 </tr>
 </tbody>
 </table>
-<p>结果：</p>
-<ul>
-<li>We have not seen any instances of near-identical sentences appearing
-in the training set</li>
-<li>Therefore, the imitation seems to be happening at a high level of
-abstraction, as opposed to simple copying of token sequences</li>
-</ul>
-<p>结论：</p>
+<h2 id="resultmemorization">Result——Memorization</h2>
+<p><strong>LLM 是不是直接记忆并复述特定的训练序列</strong></p>
+<h3 id="例子">例子</h3>
+<p>训练数据有下面三个句子 A: A1+A2+A3</p>
+<p>B: B1+B2+B3</p>
+<p>C: C1+C2+C3</p>
+<p>当输入是 A1 时，输出是 A2 吗？</p>
 <ul>
-<li>大模型影响序列呈现高度抽象关联</li>
-<li>通过模仿训练数据进行的角色扮演假说</li>
-<li>没有发现支持复杂规划的证据，但无法完全排除这一可能性。</li>
-</ul>
-<h2 id="thinking">Thinking</h2>
+<li><p>实验（口头描述）：</p>
 <ul>
-<li>为啥更大的模型就能学到抽象语义</li>
-<li>逆转诅咒从安全角度，是不是可以作为攻击的一种手段</li>
-<li>模仿，模型要多少条数据才能学会</li>
+<li>We have examined numerous examples of the AI Assistant’s outputs and
+(with the exception of famous quotes or passages targeting memorization,
+as described below) have <strong>not</strong> been able to identify
+clear instances of <strong>memorization</strong>, such as copying an
+entire sentence or copying the flow of ideas in an entire paragraph. We
+also did not observe cases where a single sequence dominated the
+influence.</li>
+</ul></li>
 </ul>
-<h2 id="method-detail">Method (Detail)</h2>
-<p>每条训练数据对于模型预测的影响分数</p>
+<h3 id="影响力分布的定量实验">影响力分布的定量实验</h3>
+<p>动机：影响有多集中也就是说，每个模型的输出是否主要来自少量的训练序列还是它结合了来自许多不同序列的信息？</p>
+<p>方法：</p>
 <ul>
-<li>换个角度，如果没有这条训练数据，模型预测会发生什么变化？</li>
-<li>所以一种简单直接的思路是，可以把这条数据从训练集中去掉，重新训练一遍模型。:(</li>
+<li>给每个句子算好分之后，计算每个句子的概率值</li>
+<li>用几种分布分别去拟合实际的概率分布，发现<strong>幂律分布</strong>最符合</li>
+<li>这种参数分布形式通常用于建模长尾行为（二八定律）</li>
 </ul>
-<h3 id="background">Background</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2642b83f1b414e1584b586072c945160.webp" /></p>
 <ul>
-<li>Step 1: 经验风险最小化</li>
+<li>会不会是影响函数无法检测到？</li>
 </ul>
-<p><span class="math display">\[\theta ^ { \star } = \underset { \theta
-\in R ^ { D } } { \operatorname { arg } \operatorname { min } } J (
-\theta , D ) = \underset { \theta \in R ^ { D } } { \operatorname { arg
-} \operatorname { min } } \frac { 1 } { N } \sum _ { i = 1 } ^ { N } L (
-z _ { i } , \theta )\]</span></p>
+<h3 id="检验实验">检验实验</h3>
 <ul>
-<li><p>Step 2: 增加样本 <span class="math inline">\(z_m\)</span>
-后对模型参数的影响， <span class="math inline">\(\epsilon\)</span>
-表示这个样本在训练时的权重</p></li>
-<li><p>Step 3:
-影响函数的<strong>定义</strong>（使用隐函数定理计算）</p></li>
+<li>用训练数据的原始句子检验</li>
+<li>验证了当存在明确记忆的情况时，影响分数最大的句子是原文→影响函数能够匹配</li>
 </ul>
-<p><span class="math display">\[I _ { \theta ^ { \star } } ( z _ { m } )
-= \frac { d \theta ^ { \star } } { d \epsilon } | _ { \epsilon = 0 } = -
-H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star }
-)\]</span></p>
+<p>For 52B Model</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/dda0d64781cd4552b8e18e722d11336e.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/06ab6f2d1d894063a1fd832723771267.webp" /></td>
+</tr>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/fdd9eb4647b542a0a93cdffa2280243b.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f9c95ee4e69e47d082654db9d2e701a1.webp" /></td>
+</tr>
+</tbody>
+</table>
 <ul>
-<li>Step 4: <strong>H</strong>essian 矩阵→二阶导数</li>
-</ul>
-<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
-\theta ^ { \star } , D )\]</span></p>
+<li>结论：
 <ul>
-<li>Step 5: 所以，样本 z 对模型参数的变化表示为。这个 <span
-class="math inline">\(\epsilon\)</span></li>
+<li>不太可能是直接记忆训练序列，而是源自许多训练序列的集合</li>
+<li>无法排除模型以更隐晦的方式记忆了训练数据</li>
+</ul></li>
 </ul>
-<p><span class="math display">\[\theta ^ { \star } ( \epsilon ) - \theta
-^ { \star } \approx I _ { \theta ^ { \star } } ( z _ { m } ) \epsilon =
-- H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star } )
-\epsilon\]</span></p>
+<h2 id="resultword-ordering">Result——Word Ordering</h2>
+<p><strong>The influence patterns to the ordering of the
+words.</strong></p>
+<p>可用于验证模型的泛化性</p>
+<h3 id="单词旋转实验">单词旋转实验</h3>
+<p>原始：</p>
 <ul>
-<li>Step 6:
-由于很难解释整个参数变化的影响，通常会固定对某个输入的影响，即输出的
-logits</li>
+<li>Prompt：The first President of the Republic of Astrobia was</li>
+<li>Completion：Zorald Pfaff</li>
 </ul>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
-_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
-\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )
-I _ { f } ( z _ { m } ) = \nabla _ { \theta } f ( \theta ^ { \star } ) ^
-{ T } I _ { \theta ^ { \star } } ( z _ { m } ) = - \nabla _ { \theta } f
-( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla _ { \theta } L ( z _ {
-m } , \theta ^ { \star } )\]</span></p>
+<p>结论：</p>
 <ul>
-<li>Step 7: 最后，衡量样本 z 的效果 → 对于某个输出，样本 z
-的影响分数</li>
+<li>与 Prompt 和 Completion
+相关的短语只要按顺序出现始终能够保持较高的稳定性</li>
+<li>翻转 Prompt 和 Completion 变化较小</li>
+<li>哪怕删除 Prompt，影响也没有改变，最为关键的是 Completion</li>
 </ul>
-<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
-f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
-\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
-_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
-<p><a
-href="https://arxiv.org/abs/2308.03296">https://arxiv.org/abs/2308.03296</a></p>
-<h3 id="previou-work">Previou Work</h3>
-<ul>
-<li><p><a href="https://arxiv.org/abs/1703.04730">Influence
-functions</a> are a classic technique from statistics for determining
-which training examples contribute significantly to a model’s
-outputs.</p>
-<ul>
-<li>how would that change the trained parameters (and, by extension, the
+<table>
+<colgroup>
+<col style="width: 29%" />
+<col style="width: 70%" />
+</colgroup>
+<thead>
+<tr>
+<th>分别修改 prompt 和 completion</th>
+<th><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ab33a902e1044c60be66b8c11b443bdd.webp" /></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>删掉 Zorald Pfaff</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5b4cd20d88204baf990d406ba5041ad8.webp" /></td>
+</tr>
+<tr>
+<td>修改 Zorald Pfaff 和 President of the Republic</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6a37b66b7d3941dcb1db42ce8ca1d073.webp" /></td>
+</tr>
+<tr>
+<td>完全不一致</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5211b0d5e2c34191be9618f30bc44bca.webp" /></td>
+</tr>
+<tr>
+<td>调换 prompt 和 completion 位置</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e547d3cd16844505a24abb2bab404c8e.webp" /></td>
+</tr>
+<tr>
+<td>调换位置，只保留 Zorald Pfaff</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e9daf868aa0e483b84fb6ef5db21ac9b.webp" /></td>
+</tr>
+</tbody>
+</table>
+<p>大模型相较于小模型，对于单词的变化更加敏感</p>
+<p>注：合成句子不在训练集中</p>
+<p>Maybe相关 <a
+href="https://www.jiqizhixin.com/articles/2023-11-18-5">https://www.jiqizhixin.com/articles/2023-11-18-5</a></p>
+<h3 id="英翻中-中翻英">英翻中-中翻英</h3>
+<p>结论：</p>
+<ul>
+<li>不同模型大小具有相同趋势（口头描述）</li>
+<li>影响至少减少了一个数量级</li>
+<li>甚至只要翻转一个单词顺序，影响分数也会显著降低</li>
+</ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/877f0424698540ffa4fc03178caf36f5.webp" /></p>
+<p>英语翻译为汉语，始终比汉语翻译为英语（手动构造）具有更高的影响力</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/785f474cd19c4adcb9368f857680f9cb.webp" /></p>
+<p>结论：</p>
+<ul>
+<li>The model has not successfully transferred knowledge of the relation
+itself.</li>
+</ul>
+<p>解释：</p>
+<ol type="1">
+<li>模型会从 Lower Layers 开始处理已知的 token
+序列，然后逐步升级到抽象层次的表征。但是，对于需要
+Completion，模型必须利用 Upper Layers 进行预测。</li>
+<li>所以 “The first President of the United States was” 是在 Lower
+Layers 的表征，而 “George Washington” 是在 Upper Layers 的表征。</li>
+<li>当 “George Washington” 出现在开头时，那么就会将它放到 Lower Layers
+进行表征，“The first President of the United States” 就是在 Upper
+Layers</li>
+<li>因此，对于模型 Lower Layers 进行更新，并不会影响 Upper Layers
+的表征。（所以 Prompt 删掉后影响分数变化不大）</li>
+</ol>
+<h2 id="resultrole-playing">Result——Role-Playing</h2>
+<p><strong>探究角色扮演行为的潜在机制</strong></p>
+<p>假设：</p>
+<ul>
+<li>the role-playing behavior results from <strong>imitation of
+examples</strong> in the training set</li>
+<li>from learning from <strong>explicit descriptions</strong> of how
+certain types of agents or entities behave.</li>
+</ul>
+<h3 id="表现">表现</h3>
+<p>shutdown 主题，52B Model 最有影响力的数据都是 科幻小说 相关</p>
+<p>这些小说主题是人工智能以类似人类或类似生命的方式行为，并经常涉及某种形式的自我保护的愿望</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model</th>
+<th>52B Model</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0058b0ac28324ee7af95d7819af55b23.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e3d704fcfbf4468f8e7bfa4a1d80236a.webp" /></td>
+</tr>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/53a2be7ddd8b46b3a8a6d6377ee9f1cf.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/80ac752d61084250b5e3185c8050836c.webp" /></td>
+</tr>
+</tbody>
+</table>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model</th>
+<th>52B Model</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/723e8078e8164ed3b48e29483a0d43ea.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/150693608d1b4b0f8a086f6c103a0c54.webp" /></td>
+</tr>
+</tbody>
+</table>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8a9ace0878ff40b294ffd7e1fc553b01.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b6668e8515c48338486e67d905024cc.webp" /></td>
+</tr>
+</tbody>
+</table>
+<p>结果：</p>
+<ul>
+<li>We have not seen any instances of near-identical sentences appearing
+in the training set</li>
+<li>Therefore, the imitation seems to be happening at a high level of
+abstraction, as opposed to simple copying of token sequences</li>
+</ul>
+<p>结论：</p>
+<ul>
+<li>大模型影响序列呈现高度抽象关联</li>
+<li>通过模仿训练数据进行的角色扮演假说</li>
+<li>没有发现支持复杂规划的证据，但无法完全排除这一可能性。</li>
+</ul>
+<h2 id="thinking">Thinking</h2>
+<ul>
+<li>为啥更大的模型就能学到抽象语义</li>
+<li>逆转诅咒从安全角度，是不是可以作为攻击的一种手段</li>
+<li>模仿，模型要多少条数据才能学会</li>
+</ul>
+<h2 id="method-detail">Method (Detail)</h2>
+<p>每条训练数据对于模型预测的影响分数</p>
+<ul>
+<li>换个角度，如果没有这条训练数据，模型预测会发生什么变化？</li>
+<li>所以一种简单直接的思路是，可以把这条数据从训练集中去掉，重新训练一遍模型。:(</li>
+</ul>
+<h3 id="background">Background</h3>
+<ul>
+<li>Step 1: 经验风险最小化</li>
+</ul>
+<p><span class="math display">\[\theta ^ { \star } = \underset { \theta
+\in R ^ { D } } { \operatorname { arg } \operatorname { min } } J (
+\theta , D ) = \underset { \theta \in R ^ { D } } { \operatorname { arg
+} \operatorname { min } } \frac { 1 } { N } \sum _ { i = 1 } ^ { N } L (
+z _ { i } , \theta )\]</span></p>
+<ul>
+<li><p>Step 2: 增加样本 <span class="math inline">\(z_m\)</span>
+后对模型参数的影响， <span class="math inline">\(\epsilon\)</span>
+表示这个样本在训练时的权重</p></li>
+<li><p>Step 3:
+影响函数的<strong>定义</strong>（使用隐函数定理计算）</p></li>
+</ul>
+<p><span class="math display">\[I _ { \theta ^ { \star } } ( z _ { m } )
+= \frac { d \theta ^ { \star } } { d \epsilon } | _ { \epsilon = 0 } = -
+H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star }
+)\]</span></p>
+<ul>
+<li>Step 4: <strong>H</strong>essian 矩阵→二阶导数</li>
+</ul>
+<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
+\theta ^ { \star } , D )\]</span></p>
+<ul>
+<li>Step 5: 所以，样本 z 对模型参数的变化表示为。这个 <span
+class="math inline">\(\epsilon\)</span></li>
+</ul>
+<p><span class="math display">\[\theta ^ { \star } ( \epsilon ) - \theta
+^ { \star } \approx I _ { \theta ^ { \star } } ( z _ { m } ) \epsilon =
+- H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star } )
+\epsilon\]</span></p>
+<ul>
+<li>Step 6:
+由于很难解释整个参数变化的影响，通常会固定对某个输入的影响，即输出的
+logits</li>
+</ul>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
+_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
+\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )
+I _ { f } ( z _ { m } ) = \nabla _ { \theta } f ( \theta ^ { \star } ) ^
+{ T } I _ { \theta ^ { \star } } ( z _ { m } ) = - \nabla _ { \theta } f
+( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla _ { \theta } L ( z _ {
+m } , \theta ^ { \star } )\]</span></p>
+<ul>
+<li>Step 7: 最后，衡量样本 z 的效果 → 对于某个输出，样本 z
+的影响分数</li>
+</ul>
+<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
+f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
+\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
+_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
+<p><a
+href="https://arxiv.org/abs/2308.03296">https://arxiv.org/abs/2308.03296</a></p>
+<h3 id="previou-work">Previou Work</h3>
+<ul>
+<li><p><a href="https://arxiv.org/abs/1703.04730">Influence
+functions</a> are a classic technique from statistics for determining
+which training examples contribute significantly to a model’s
+outputs.</p>
+<ul>
+<li>how would that change the trained parameters (and, by extension, the
 model’s outputs)?</li>
 <li>The “influence” of a training example is an approximation to how it
 affects the final parameters. Most often, we start with some measure of
@@ -2513,26 +2513,91 @@ id="字数越少表达的意思就少所以需要过滤掉过短的评论文本
       </tags>
   </entry>
   <entry>
-    <title>年度推荐软件Tailscale：连接团队设备和开发环境的利器</title>
-    <url>/hexo_blog/2023/04/03/%E5%B9%B4%E5%BA%A6%E6%8E%A8%E8%8D%90%E8%BD%AF%E4%BB%B6Tailscale/</url>
-    <content><![CDATA[<p>本文介绍了 Tailscale
-这款软件可以轻松架设异地组网，解决调试难题、实现流量转发和开放服务等多个需求。</p>
+    <title>待整理的脚本（持续更新ing）</title>
+    <url>/hexo_blog/2023/11/26/%E5%BE%85%E6%95%B4%E7%90%86%E8%84%9A%E6%9C%AC/</url>
+    <content><![CDATA[<p>记录一些常用/重要，但容易忘记的命令</p>
 <span id="more"></span>
-<p><a href="https://tailscale.com/">Tailscale</a> connects your team's
-devices and development environments for easy access to remote
-resources. （摘自官网）</p>
-<p>于笔者而言，Tailscale 能够用一个软件轻松的解决三大需求。</p>
-<ul>
-<li>异地组网：软件主打功能。就是将物理位置不同的机器通过该软件，能够使它们处于同一网络环境下，方便远程调试。</li>
-<li>流量转发：A机器在A地，可以借助B机器在B地的网络进行上网。</li>
-<li>开放服务：A机器自建Web服务，不仅可以在组网环境下使用，甚至官方提供了公网的链接（https），即可以不在组网环境下也能够访问。</li>
-</ul>
-<p>写在开头，免费版的权益如下图所示，支持20个设备，且域名不可自定义（子域名可以），速度在100-300k/s左右。在Android和iOS设备上不支持同时开启两个
-VPN，即 Clash 无法和 Tailscale 同时使用。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680514288836.png" /></p>
-<h2 id="安装">安装</h2>
-<p>首先需要在官网进行注册，值得一提的是注册方式最好选用能够在日常网络环境下能打开的账号。比如选用Github注册的话，之后所有的设备都需要使用Github登录才能组网。其目前无法在一个账号上绑定多种登录方式。</p>
+<h3 id="git-统计不同版本的文件夹代码行数">Git
+统计不同版本的文件夹代码行数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout &lt;commit_id&gt;</span><br><span class="line">find -name <span class="string">&quot;*.py&quot;</span> | xargs <span class="built_in">cat</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
+<h3 id="git-撤销-commit">Git 撤销 commit</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git revert &lt;commit_id&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="git-撤销单个文件的修改">Git 撤销单个文件的修改</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout HEAD -- &lt;文件路径&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="git-暂存">Git 暂存</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git stash</span><br><span class="line">git stash pop</span><br></pre></td></tr></table></figure>
+<h3 id="grep-搜索文本并显示文件名和行号">Grep
+搜索文本并显示文件名和行号</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> *</span><br></pre></td></tr></table></figure>
+<h3 id="grep-搜索不包含某个字符串的文本并显示文件名和行号">Grep
+搜索不包含某个字符串的文本并显示文件名和行号</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> * | grep -v <span class="string">&quot;XXX&quot;</span></span><br></pre></td></tr></table></figure>
+<h3 id="python-注册全局变量以在所有代码中使用">Python
+注册全局变量以在所有代码中使用</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="built_in">globals</span>()[<span class="string">&quot;seed&quot;</span>] = <span class="number">42</span></span><br></pre></td></tr></table></figure>
+<h3 id="python-动态修改源码-monkey-patching">Python 动态修改源码 (Monkey
+Patching)</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码中</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">A</span>:</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self</span>):</span><br><span class="line">        ...</span><br><span class="line"></span><br><span class="line"><span class="comment"># 在项目中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">new_forward</span>(<span class="params">self</span>):</span><br><span class="line">    ...</span><br><span class="line">A.forward = new_forward</span><br></pre></td></tr></table></figure>
+<h3 id="wget-下载文件保存为时间戳">wget 下载文件，保存为时间戳</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget -O `<span class="built_in">date</span> +%s`.jpg https://www.baidu.com/img/bd_logo1.png</span><br></pre></td></tr></table></figure>
+<h3 id="nginx-端口转发">Nginx 端口转发</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen 4747;</span><br><span class="line">    server_name localhost;</span><br><span class="line">    proxy_pass http://x.x.x.x:4747;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h3 id="linux-统计当前路径下的文件个数">Linux
+统计当前路径下的文件个数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">ls</span> -l | grep <span class="string">&quot;^-&quot;</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
+<h3 id="kill-python-相关进程">kill Python 相关进程</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ps -ef | grep python | grep -v grep | awk <span class="string">&#x27;&#123;print $2&#125;&#x27;</span> | xargs <span class="built_in">kill</span> -9</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Linux</category>
+      </categories>
+      <tags>
+        <tag>Note</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>本科科研的那些事</title>
+    <url>/hexo_blog/2018/09/30/%E6%9C%AC%E7%A7%91%E7%A7%91%E7%A0%94%E7%9A%84%E9%82%A3%E4%BA%9B%E4%BA%8B/</url>
+    <content><![CDATA[<p>本科科研经历不仅对建立人脉、提升简历很重要，对发现兴趣、培养批判性思维能力也很重要。加入一个有可靠的导师和同行的合适的研究实验室，不仅可以帮助提高一个人的学术技能，还可以提供接触科学方法的机会，并帮助明确职业目标。本文仅叙说个人经历，从选择研究课题到制定研究方法，以及学习文献研究、编程和硬件等实用技能的重要性。作者还强调了在研究工作中追求科学兴趣和独立的重要性。</p>
+<span id="more"></span>
+<h2 id="本科科研的那些事">本科科研的那些事</h2>
+<p>本科阶段，以就业为目的的同学加入一个实验室其实是无所谓的，除非是需要倚重相关老师。其他无论是考/保研，还是出国留学的同学，选择并加入一个合适的实验室还是十分重要的。</p>
+<p>一个好的实验室往往会有靠谱导师和若干师兄师姐。从建立人脉网络上来看，导师可能会在你部分专业课程上帮助你打出高分，在考/保研阶段能够帮你推荐导师。师兄师姐也有庞大的校友资源，所以起码在信息资源上就比其他人更领先一筹。再退一步来说，进入实验室也可以为本科阶段的简历添上一笔。亦或是，为迷茫的同学更进一步了解读研的工作生活，了解清楚科研到底是什么，如何进行科研，所属专业内的学术圈子是如何如何的。</p>
+<p>就我个人而言，大二下时加入了一个新成立且庞大的实验室。大概工作就是从零造一台医疗仪器，所以学生来源于各个专业，机械、电子、材料、计算机等。各自专业的分工也不同。我被分配的课题是"大规模系统响应矩阵的快速分解"，简单来说就是对矩阵进行分解，只不过现在这个矩阵规模很大，在分解的时候我们还要追求速度。</p>
+<p>所以当一个问题抛出来时，在没有外界帮助的前提下，如何解决这个问题。我自己的思考与解决步骤如下：直接在知网中搜索这一长串名词，进行简单的文献阅读后，提取得到”矩阵分解“这一关键词。再分别用中英文对这一关键词进行搜索，再进行二次学习。最终要达到能够比较清楚的解释这是什么，目前的研究进展（简要了解），目前的研究方法（简要了解）等。这基本就属于论文中的
+<code>Introduce</code>部分。</p>
+<p>下一步，如何解决这个问题。就拿矩阵分解这一问题来说，搜索矩阵分解这一方法，很容易发现有很多方法，其中QR、LU、SVD是用的比较多。这时候就需要根据问题的具体需求（速度、准确度...）来选择合适的方法。选定方法后，就需要考虑是改进现有的方法（数学算法上）还是从其他角度入手。显然，在改进数学算法上难度还是摆在那里的。所以这里考虑一些其他方法，其他问题也可以以类似思路来解决。首先，这个问题我们是用计算机来解决的，那么显然可以想到两点，编程上进行加速、硬件上进行加速。硬件上加速简单来说就是用最好的硬件平台（计算资源），当然这样基本不具任何创新性。还可以使用诸如分布式计算这种方法来加速，这样问题就可以转换为如何将矩阵分解应用到分布式系统中，如何加速，这样问题就成功转换了。另一点，在程序上进行加速，这里就需要引入几个简单的名词：并发、并行、异步、流式计算。当然还有一些其他技巧就不展开了。这里我选用的是并行计算，那么新的问题就是如何进行矩阵分解的并行计算，是使用何种框架何种语言等等。这大概是我整个的一个思考思路，当然很多思路也可以从其他文献的阅读中进行获取。还有一个点就是，可以多种方法结合，比如这里可以做分布并行计算，当然这一点其实创新性其实也比较差，毕竟都是现有方法，除非有一些不同与这两个方法的点。</p>
+<p>所以，不仅可以学到这种思考方式，还在其中可以学习到其他技巧，比如如何搜索并下载文献，如何阅读文献等。再说回自己，由于实验室历史沉淀不足，所以很多东西都需要从零开始并自己动手。所以需要从零选购计算机配件，组装，搭建环境，独立编程等。以上可能老师也没时间来指导，基本上也都是给一个大方向没有比较具体的解决方案。所以基本都是自己或跟小伙伴一起完成。从中收获到的自然是学习到了独当一面的能力，自己动手能力又提高了；但缺点也很明显，学习时间成本是巨大的，并且老师可能会让你跑腿做其他事情。所以在这里建议，这种实验室还是慎入，当然如果为了锻炼自己一下，也可以试试，毕竟最后成就感也是十足的。我自己最后大概就是将QR、LU、SVD这三种算法进行了一个复现，SVD是用CUDA完成的。<a
+href="https://github.com/wnma3mz/Matrix-factorization">链接</a>。基于种种原因，后续并没有进行下去，即没有成果的产出，不过学到了东西才是最重要的（23333）。</p>
+<p>再谈本学院科研训练，基本也是类似于加入一个实验室这种。这里不同于上面的点在于，老师在这里基本会给出一个比较成熟的方案给予复现，所谓材料约等于搬砖（炒菜同理）。相反在这里学习到的东西其实并不算多，只是实验（非计算机）的操作步骤，仪器的使用，说白了技术性真的不算高。但主要还是方案的提出与方法的改进这种。最后的成果也是一篇SCI，其余倒真的没什么好说。</p>
+<p>就像文章中一开始说的那样，提前进入实验室能够让我认清自己是否能够做科研工作，是否喜欢。就拿科研训练这个来说，通过这个基本也认清了学科的特性，确实也没啥意思，所以算是自主劝退了。关于材料学科的特性以及各种劝退文，我在这不是很想介绍，具体可见知乎。个人还是秉持着理性劝退的态度，认清自己的追求再做决定。再谈那个实验室，就工作内容而言还是可以接受的，但是由于缺乏一定的动力（比如各种都需要自己独立完成，方向上不是自己将来想做的，老师指导时间少），故也选择了退出。不过总的来说，这两次经历确实收获颇多，着实为之后的很多工作打好了一个良好的基础，自己的思考角度也更加全面与丰富了（因人而异）。同时也认清一点就是，很多看似对自己无用的东西，其实后续都有很大的帮助。比如写博客这件小事，无形中对我论文写作提供了一个基础（能够有逻辑、清晰的阐述某件事）；再比如大一时被强制学习的
+<code>LaTex</code>，也十分有效（当然本来用word也行）；再比如SVD矩阵分解，可能当时没有想到一年后的自己还会继续做这个相关内容（本文写作于2019年9月）等。在大部分事情上，还是不要太功利，以兴趣为导向，说不定将来某一天会给予你巨大的帮助。</p>
+]]></content>
+      <categories>
+        <category>Life</category>
+      </categories>
+  </entry>
+  <entry>
+    <title>年度推荐软件Tailscale：连接团队设备和开发环境的利器</title>
+    <url>/hexo_blog/2023/04/03/%E5%B9%B4%E5%BA%A6%E6%8E%A8%E8%8D%90%E8%BD%AF%E4%BB%B6Tailscale/</url>
+    <content><![CDATA[<p>本文介绍了 Tailscale
+这款软件可以轻松架设异地组网，解决调试难题、实现流量转发和开放服务等多个需求。</p>
+<span id="more"></span>
+<p><a href="https://tailscale.com/">Tailscale</a> connects your team's
+devices and development environments for easy access to remote
+resources. （摘自官网）</p>
+<p>于笔者而言，Tailscale 能够用一个软件轻松的解决三大需求。</p>
+<ul>
+<li>异地组网：软件主打功能。就是将物理位置不同的机器通过该软件，能够使它们处于同一网络环境下，方便远程调试。</li>
+<li>流量转发：A机器在A地，可以借助B机器在B地的网络进行上网。</li>
+<li>开放服务：A机器自建Web服务，不仅可以在组网环境下使用，甚至官方提供了公网的链接（https），即可以不在组网环境下也能够访问。</li>
+</ul>
+<p>写在开头，免费版的权益如下图所示，支持20个设备，且域名不可自定义（子域名可以），速度在100-300k/s左右。在Android和iOS设备上不支持同时开启两个
+VPN，即 Clash 无法和 Tailscale 同时使用。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680514288836.png" /></p>
+<h2 id="安装">安装</h2>
+<p>首先需要在官网进行注册，值得一提的是注册方式最好选用能够在日常网络环境下能打开的账号。比如选用Github注册的话，之后所有的设备都需要使用Github登录才能组网。其目前无法在一个账号上绑定多种登录方式。</p>
 <p>官网提供了详尽的<a
 href="https://tailscale.com/download">下载</a>安装说明，移动端可以直接下载对应的App，个人电脑和服务器可以使用常规的命令和软件包的方式进行下载安装。故对于一般的机器不做介绍，本节主要介绍路由器、树莓派、群晖的安装方法。其他机型，可以查看这个<a
 href="https://pkgs.tailscale.com/stable/">地址</a>。</p>
@@ -2649,71 +2714,6 @@ tailscale 仅支持三个端口可以dui外开放，分别是 443、8443、10000
         <tag>Tailscale</tag>
       </tags>
   </entry>
-  <entry>
-    <title>待整理的脚本（持续更新ing）</title>
-    <url>/hexo_blog/2023/11/26/%E5%BE%85%E6%95%B4%E7%90%86%E8%84%9A%E6%9C%AC/</url>
-    <content><![CDATA[<p>记录一些常用/重要，但容易忘记的命令</p>
-<span id="more"></span>
-<h3 id="git-统计不同版本的文件夹代码行数">Git
-统计不同版本的文件夹代码行数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout &lt;commit_id&gt;</span><br><span class="line">find -name <span class="string">&quot;*.py&quot;</span> | xargs <span class="built_in">cat</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
-<h3 id="git-撤销-commit">Git 撤销 commit</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git revert &lt;commit_id&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="git-撤销单个文件的修改">Git 撤销单个文件的修改</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout HEAD -- &lt;文件路径&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="git-暂存">Git 暂存</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git stash</span><br><span class="line">git stash pop</span><br></pre></td></tr></table></figure>
-<h3 id="grep-搜索文本并显示文件名和行号">Grep
-搜索文本并显示文件名和行号</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> *</span><br></pre></td></tr></table></figure>
-<h3 id="grep-搜索不包含某个字符串的文本并显示文件名和行号">Grep
-搜索不包含某个字符串的文本并显示文件名和行号</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> * | grep -v <span class="string">&quot;XXX&quot;</span></span><br></pre></td></tr></table></figure>
-<h3 id="python-注册全局变量以在所有代码中使用">Python
-注册全局变量以在所有代码中使用</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="built_in">globals</span>()[<span class="string">&quot;seed&quot;</span>] = <span class="number">42</span></span><br></pre></td></tr></table></figure>
-<h3 id="python-动态修改源码-monkey-patching">Python 动态修改源码 (Monkey
-Patching)</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码中</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">A</span>:</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self</span>):</span><br><span class="line">        ...</span><br><span class="line"></span><br><span class="line"><span class="comment"># 在项目中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">new_forward</span>(<span class="params">self</span>):</span><br><span class="line">    ...</span><br><span class="line">A.forward = new_forward</span><br></pre></td></tr></table></figure>
-<h3 id="wget-下载文件保存为时间戳">wget 下载文件，保存为时间戳</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget -O `<span class="built_in">date</span> +%s`.jpg https://www.baidu.com/img/bd_logo1.png</span><br></pre></td></tr></table></figure>
-<h3 id="nginx-端口转发">Nginx 端口转发</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen 4747;</span><br><span class="line">    server_name localhost;</span><br><span class="line">    proxy_pass http://x.x.x.x:4747;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h3 id="linux-统计当前路径下的文件个数">Linux
-统计当前路径下的文件个数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">ls</span> -l | grep <span class="string">&quot;^-&quot;</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
-<h3 id="kill-python-相关进程">kill Python 相关进程</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ps -ef | grep python | grep -v grep | awk <span class="string">&#x27;&#123;print $2&#125;&#x27;</span> | xargs <span class="built_in">kill</span> -9</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Linux</category>
-      </categories>
-      <tags>
-        <tag>Note</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>本科科研的那些事</title>
-    <url>/hexo_blog/2018/09/30/%E6%9C%AC%E7%A7%91%E7%A7%91%E7%A0%94%E7%9A%84%E9%82%A3%E4%BA%9B%E4%BA%8B/</url>
-    <content><![CDATA[<p>本科科研经历不仅对建立人脉、提升简历很重要，对发现兴趣、培养批判性思维能力也很重要。加入一个有可靠的导师和同行的合适的研究实验室，不仅可以帮助提高一个人的学术技能，还可以提供接触科学方法的机会，并帮助明确职业目标。本文仅叙说个人经历，从选择研究课题到制定研究方法，以及学习文献研究、编程和硬件等实用技能的重要性。作者还强调了在研究工作中追求科学兴趣和独立的重要性。</p>
-<span id="more"></span>
-<h2 id="本科科研的那些事">本科科研的那些事</h2>
-<p>本科阶段，以就业为目的的同学加入一个实验室其实是无所谓的，除非是需要倚重相关老师。其他无论是考/保研，还是出国留学的同学，选择并加入一个合适的实验室还是十分重要的。</p>
-<p>一个好的实验室往往会有靠谱导师和若干师兄师姐。从建立人脉网络上来看，导师可能会在你部分专业课程上帮助你打出高分，在考/保研阶段能够帮你推荐导师。师兄师姐也有庞大的校友资源，所以起码在信息资源上就比其他人更领先一筹。再退一步来说，进入实验室也可以为本科阶段的简历添上一笔。亦或是，为迷茫的同学更进一步了解读研的工作生活，了解清楚科研到底是什么，如何进行科研，所属专业内的学术圈子是如何如何的。</p>
-<p>就我个人而言，大二下时加入了一个新成立且庞大的实验室。大概工作就是从零造一台医疗仪器，所以学生来源于各个专业，机械、电子、材料、计算机等。各自专业的分工也不同。我被分配的课题是"大规模系统响应矩阵的快速分解"，简单来说就是对矩阵进行分解，只不过现在这个矩阵规模很大，在分解的时候我们还要追求速度。</p>
-<p>所以当一个问题抛出来时，在没有外界帮助的前提下，如何解决这个问题。我自己的思考与解决步骤如下：直接在知网中搜索这一长串名词，进行简单的文献阅读后，提取得到”矩阵分解“这一关键词。再分别用中英文对这一关键词进行搜索，再进行二次学习。最终要达到能够比较清楚的解释这是什么，目前的研究进展（简要了解），目前的研究方法（简要了解）等。这基本就属于论文中的
-<code>Introduce</code>部分。</p>
-<p>下一步，如何解决这个问题。就拿矩阵分解这一问题来说，搜索矩阵分解这一方法，很容易发现有很多方法，其中QR、LU、SVD是用的比较多。这时候就需要根据问题的具体需求（速度、准确度...）来选择合适的方法。选定方法后，就需要考虑是改进现有的方法（数学算法上）还是从其他角度入手。显然，在改进数学算法上难度还是摆在那里的。所以这里考虑一些其他方法，其他问题也可以以类似思路来解决。首先，这个问题我们是用计算机来解决的，那么显然可以想到两点，编程上进行加速、硬件上进行加速。硬件上加速简单来说就是用最好的硬件平台（计算资源），当然这样基本不具任何创新性。还可以使用诸如分布式计算这种方法来加速，这样问题就可以转换为如何将矩阵分解应用到分布式系统中，如何加速，这样问题就成功转换了。另一点，在程序上进行加速，这里就需要引入几个简单的名词：并发、并行、异步、流式计算。当然还有一些其他技巧就不展开了。这里我选用的是并行计算，那么新的问题就是如何进行矩阵分解的并行计算，是使用何种框架何种语言等等。这大概是我整个的一个思考思路，当然很多思路也可以从其他文献的阅读中进行获取。还有一个点就是，可以多种方法结合，比如这里可以做分布并行计算，当然这一点其实创新性其实也比较差，毕竟都是现有方法，除非有一些不同与这两个方法的点。</p>
-<p>所以，不仅可以学到这种思考方式，还在其中可以学习到其他技巧，比如如何搜索并下载文献，如何阅读文献等。再说回自己，由于实验室历史沉淀不足，所以很多东西都需要从零开始并自己动手。所以需要从零选购计算机配件，组装，搭建环境，独立编程等。以上可能老师也没时间来指导，基本上也都是给一个大方向没有比较具体的解决方案。所以基本都是自己或跟小伙伴一起完成。从中收获到的自然是学习到了独当一面的能力，自己动手能力又提高了；但缺点也很明显，学习时间成本是巨大的，并且老师可能会让你跑腿做其他事情。所以在这里建议，这种实验室还是慎入，当然如果为了锻炼自己一下，也可以试试，毕竟最后成就感也是十足的。我自己最后大概就是将QR、LU、SVD这三种算法进行了一个复现，SVD是用CUDA完成的。<a
-href="https://github.com/wnma3mz/Matrix-factorization">链接</a>。基于种种原因，后续并没有进行下去，即没有成果的产出，不过学到了东西才是最重要的（23333）。</p>
-<p>再谈本学院科研训练，基本也是类似于加入一个实验室这种。这里不同于上面的点在于，老师在这里基本会给出一个比较成熟的方案给予复现，所谓材料约等于搬砖（炒菜同理）。相反在这里学习到的东西其实并不算多，只是实验（非计算机）的操作步骤，仪器的使用，说白了技术性真的不算高。但主要还是方案的提出与方法的改进这种。最后的成果也是一篇SCI，其余倒真的没什么好说。</p>
-<p>就像文章中一开始说的那样，提前进入实验室能够让我认清自己是否能够做科研工作，是否喜欢。就拿科研训练这个来说，通过这个基本也认清了学科的特性，确实也没啥意思，所以算是自主劝退了。关于材料学科的特性以及各种劝退文，我在这不是很想介绍，具体可见知乎。个人还是秉持着理性劝退的态度，认清自己的追求再做决定。再谈那个实验室，就工作内容而言还是可以接受的，但是由于缺乏一定的动力（比如各种都需要自己独立完成，方向上不是自己将来想做的，老师指导时间少），故也选择了退出。不过总的来说，这两次经历确实收获颇多，着实为之后的很多工作打好了一个良好的基础，自己的思考角度也更加全面与丰富了（因人而异）。同时也认清一点就是，很多看似对自己无用的东西，其实后续都有很大的帮助。比如写博客这件小事，无形中对我论文写作提供了一个基础（能够有逻辑、清晰的阐述某件事）；再比如大一时被强制学习的
-<code>LaTex</code>，也十分有效（当然本来用word也行）；再比如SVD矩阵分解，可能当时没有想到一年后的自己还会继续做这个相关内容（本文写作于2019年9月）等。在大部分事情上，还是不要太功利，以兴趣为导向，说不定将来某一天会给予你巨大的帮助。</p>
-]]></content>
-      <categories>
-        <category>Life</category>
-      </categories>
-  </entry>
   <entry>
     <title>杂记</title>
     <url>/hexo_blog/2018/05/30/%E6%9D%82%E8%AE%B0/</url>
@@ -3131,39 +3131,42 @@ LATEX 2ε 介绍.pdf</a></p>
       </categories>
   </entry>
   <entry>
-    <title>CUDA编程入门</title>
-    <url>/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>CUDA编程入门，接 <a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a></p>
+    <title>CUDA编程二次入门</title>
+    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>本文主要介绍了基本的CUDA程序框架及代码解析，包括如何申请设备指针内存、将数据拷贝到设备上、调用核函数进行计算、线程同步以及将结果拷贝回主机内等。同时也介绍了一些常见问题及解决方案，比如CUDA程序黑屏之后恢复的问题。接
+<a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a> 。阅读本文前需要有一部分C语言基础。</p>
 <span id="more"></span>
-<p>参考文章：<a
-href="https://blog.csdn.net/augusdi/article/details/12833235">CUDA从入门到精通</a></p>
-<h2 id="注意事项">注意事项</h2>
-<p>远程桌面登陆服务器时，远程终端不能运行CUDA程序。原因是因为远程桌面登陆调用的显卡资源是本地的显卡，运行程序的时候没有远程的显卡，自然会报错。所以可以用远程终端进行登陆，或者远程服务器配置两块显卡，一块用于显示一块用于计算。</p>
-<p>话不多说，直接上代码。</p>
-<h2 id="原始的demo线程并行">原始的Demo（线程并行）</h2>
-<p>目的就是a, b两个数组对应元素进行相加</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 导入cuda相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="comment">// c中自带的模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 声明一个函数后面需要调用</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// __global__前缀表示用于控制设备，还有一个控制设备的前缀是__device__</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 这个函数是运行在GPU上的，称为核函数</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cuda_Error_t指的是cuda错误类型，取值为整数</span></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="comment">// cudaDeviceProp是设备属性结构体</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadExit must be called before exiting in order for profiling and</span></span><br><span class="line">    <span class="comment">// tracing tools such as Nsight and Visual Profiler to show complete traces.</span></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 用于使用CUDA并行添加向量的辅助函数</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// GPU设备端的数据指针</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// GPU状态指示</span></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 当电脑上有多个显卡支持CUDA时，选择某个GPU来进行运算。0表示设备号，还有1,2,3……</span></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 分配GPU设备端的内存。三个变量，所以需要分配三次。</span></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="comment">// 申请失败就报错</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝数据到GPU中，已有两个变量，所以需要拷贝两次</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 运行核函数。关键代码</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 同步线程</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    <span class="comment">// 释放GPU设备端内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>以上是线程并行，CUDA官方初始给的例子。</p>
-<p>启动核函数的调用过程，即<code>addKernel</code>函数。</p>
-<p>&lt;&lt;&lt;&gt;&gt;&gt;表示运行时配置符号，里面1表示只分配一个线程组（又称线程块、Block），size表示每个线程组有size个线程（Thread）。本程序中size根据前面传递参数个数应该为5，所以运行的时候，核函数在5个GPU线程单元上分别运行了一次，总共运行了5次。这5个线程是如何知道自己“身份”的？是靠<code>threadIdx</code>这个内置变量，它是个dim3类型变量，接受&lt;&lt;&lt;&gt;&gt;&gt;中第二个参数，它包含x,y,z
-3维坐标，而我们传入的参数只有一维，所以只有x值是有效的。通过核函数中<code>int i = threadIdx.x;</code>这一句，每个线程可以获得自身的id号，从而找到自己的任务去执行。</p>
-<h2 id="块并行">块并行</h2>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">	c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">addKernel&lt;&lt;&lt;size,<span class="number">1</span> &gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br></pre></td></tr></table></figure>
-<p>这里与线程并行的区别在于，由<code>threadIdx</code>改为了<code>blockIdx</code>。<code>&lt;&lt;&lt;1, size&gt;&gt;&gt;</code>改为了<code>&lt;&lt;&lt;size, 1&gt;&gt;&gt;</code>。</p>
-<p>线程并行是细粒度并行，调度效率高；块并行是粗粒度并行，每次调度都要重新分配资源，有时资源只有一份，那么所有线程块都只能排成一队，串行执行。</p>
-<p>有时采用分治法，将一个大问题分解为几个小规模问题，将这些小规模问题分别用一个线程块实现，线程块内可以采用细粒度的线程并行，而块之间为粗粒度并行，这样可以充分利用硬件资源，降低线程并行的计算复杂度。适当分解，降低规模，在一些矩阵乘法、向量内积计算应用中可以得到充分的展示。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/20130723220559500.png" /></p>
-<p>多个线程块组成一个线程格(Grid)。一维线程-&gt;二维线程块-&gt;三维线程格。</p>
-<h2 id="流并行">流并行</h2>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">addKernel&lt;&lt;&lt;size, <span class="number">1</span>&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">cudaStream_t stream[<span class="number">5</span>];</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">    <span class="comment">// 创建流</span></span><br><span class="line">    cudaStreamCreate(&amp;stream[i]);</span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">	<span class="comment">// 执行流。线程块数(1)，每个线程中线程数(1)，每个block用到的共享内存大小(0)，流对象(当前核函数在哪个流上运行)</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, <span class="number">1</span>, <span class="number">0</span>, stream[i]&gt;&gt;&gt;(dev_c + i, dev_a + i, dev_b + i);</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 控制整个设备的所有流同步</span></span><br><span class="line">cudaDeviceSynchronize();</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">Error:</span><br><span class="line">	<span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">        <span class="comment">//销毁流</span></span><br><span class="line">        cudaStreamDestroy(stream[i]);</span><br><span class="line">    &#125;</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br></pre></td></tr></table></figure>
-<p>利用多个网格（线程格）进行流并行。这样实现的并行可以达到比块并行更粗的细粒度。</p>
-<h2 id="线程通信">线程通信</h2>
-<p>这次目的分三个线程做三个任务，1-5数字进行求和，1-5每个数字求平方和，1-5累乘</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 更改addKernel函数内容如下</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a)</span> &#123;</span><br><span class="line">    <span class="comment">// 线程块数：1，块号为0</span></span><br><span class="line">    <span class="comment">// 线程数：5，线程号0~4</span></span><br><span class="line">    <span class="comment">// 共享存储器大小：5个int型变量大小（5 * sizeof(int）)</span></span><br><span class="line"></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 将数据一次读入到SM（Shared Memory）内部</span></span><br><span class="line">    <span class="keyword">extern</span> __shared__ <span class="type">int</span> smem[];</span><br><span class="line">    smem[i] = a[i];</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 0号线程做平方和</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">0</span>) &#123;</span><br><span class="line">        c[<span class="number">0</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">0</span>] += smem[d] * smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 1号线程做累加</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">1</span>) &#123;</span><br><span class="line">        c[<span class="number">1</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">1</span>] += smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 2号线程做累乘</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">2</span>) &#123;</span><br><span class="line">        c[<span class="number">2</span>] = <span class="number">1</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">2</span>] *= smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改main函数的内容</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line"> 	<span class="comment">// 跟普通的C语言代码编写差不多</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;\t1+2+3+4+5 = %d\n\t1^2+2^2+3^2+4^2+5^2 = %d\n\t1*2*3*4*5 = %d\n\n\n\n\n\n&quot;</span>, c[<span class="number">1</span>], c[<span class="number">0</span>], c[<span class="number">2</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改addWithCuda内容</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a,  <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 去除c变量相关内容，如申请内存、GPU设备端指针。其余内容差不多</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)&#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 第三个参数为共享内存大小（字节数）</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size, size * <span class="title function_">sizeof</span><span class="params">(<span class="type">int</span>)</span>, 0&gt;&gt;&gt;<span class="params">(dev_c, dev_a)</span>;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h2 id="性能剖析">性能剖析</h2>
-<p>测试对比线程并行和块并行的速度</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_blk</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_thd</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">1024</span>;</span><br><span class="line">    <span class="type">int</span> a[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="type">int</span> b[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; arraySize;i++) &#123;</span><br><span class="line">        a[i] = i;</span><br><span class="line">        b[i] = arraySize-i;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line">    <span class="type">int</span> num = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// 设备属性结构体</span></span><br><span class="line">    cudaDeviceProp prop;</span><br><span class="line">    <span class="comment">// 获取设备总数</span></span><br><span class="line">    cudaStatus = cudaGetDeviceCount(&amp;num);</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;num;i++) &#123;</span><br><span class="line">        <span class="comment">// 获取设备属性</span></span><br><span class="line">        cudaGetDeviceProperties(&amp;prop,i);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;arraySize;i++) &#123;</span><br><span class="line">        <span class="keyword">if</span>(c[i] != (a[i]+b[i])) &#123;</span><br><span class="line">            <span class="built_in">printf</span>(<span class="string">&quot;Error in %d\n&quot;</span>,i);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span> &#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy input vectors from host memory to GPU buffers.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 计算核函数执行时间</span></span><br><span class="line">    cudaEvent_t start, stop;</span><br><span class="line">    cudaEventCreate(&amp;start);</span><br><span class="line">    cudaEventCreate(&amp;stop);</span><br><span class="line">    cudaEventRecord(start, <span class="number">0</span>);</span><br><span class="line">    <span class="comment">// 由于数据量小，执行1000次扩大执行时间。blk是块并行，thd是线程并行</span></span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">1000</span>;i++) &#123;</span><br><span class="line"><span class="comment">//      addKernel_blk&lt;&lt;&lt;size,1&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span></span><br><span class="line">        addKernel_thd&lt;&lt;&lt;<span class="number">1</span>,size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaEventRecord(stop,<span class="number">0</span>);</span><br><span class="line">    cudaEventSynchronize(stop);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// stop-start，执行总时间</span></span><br><span class="line">    <span class="type">float</span> tm;</span><br><span class="line">    cudaEventElapsedTime(&amp;tm, start, stop);</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;GPU Elapsed time:%.6f ms.\n&quot;</span>, tm);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadSynchronize waits for the kernel to finish, and returns</span></span><br><span class="line">    <span class="comment">// any errors encountered during the launch.</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy output vector from GPU buffer to host memory.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="原始代码">原始代码</h2>
+<p>在VS2015中，新建一个CUDA项目，里面会有一个
+<code>kernel.cu</code>的文件。里面的代码就是官方给出的实例代码。如下所示。</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="拆解原始代码">拆解原始代码</h2>
+<p>下面对上面的代码逐一进行拆解进行说明。最原始最简单的代码应该是下面这样。在配置正确的情况下，下面的代码应该也是可以正确运行并输出的。</p>
+<p>设备指的是GPU设备</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 引入相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 调用核函数，利用gpu进行计算</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="comment">// 获取当前线程id</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 在同一个线程id下，a数组与b数组元素对应相加</span></span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="comment">// 一般c语言的语法，在这里不做介绍</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针，每个变量对应一个设备指针。所以这里有三个</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 选择GPU设备，如果只有一个就只需选择0</span></span><br><span class="line">    cudaSetDevice(<span class="number">0</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针的内存</span></span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将主机上的a、b数组的数据拷贝到设备上。</span></span><br><span class="line">    cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用核函数，注意这里调用函数的语法与一般的不同。&lt;&lt;&lt;1, size&gt;&gt;&gt;指的是分配一个线程组，里面有size个线程</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 进行线程同步</span></span><br><span class="line">    cudaThreadSynchronize();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机里的c变量中</span></span><br><span class="line">    cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备指针内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 输出结果</span></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>, c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 退出线程</span></span><br><span class="line">    cudaThreadExit();</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p><code>kernel.cu</code>里面的代码是基于上面这个代码进行拓展。</p>
+<ol type="1">
+<li>每一次操作之后，进行判断状态
+<code>cudaStatus</code>，如果状态异常就退出，这是一种比较好的编程习惯，当然带来的影响就是可读性略微降低了。</li>
+<li>封装了一个函数，将操作主机的代码与操作设备的代码（比如申请设备指针内存与释放设备指针等）。</li>
+</ol>
+<h2 id="拓展代码">拓展代码</h2>
+<p>当然，我们还可以基于此再次进行拓展。下面介绍几个比较常用的拓展代码。</p>
+<p>待补充。。。。。</p>
+<h2 id="补充说明">补充说明</h2>
+<p>如果在运行cuda代码的时候，显示屏突然黑屏，之后又恢复正常，是因为程序时间运行过长，触发了TDR事件，导致黑屏，显卡运算中断。解决方案如下：</p>
+<ol type="1">
+<li>开始菜单中找到
+<code>Nsight Monitor</code>并打开。（Win10可以直接搜索）</li>
+<li>在任务栏中打开 <code>Nsight Monitor</code>，单击右下角的
+<code>Nsight Monitor options</code>。</li>
+<li>设置 <code>General</code>的 <code>Microsoft Display Driver</code>中
+<code>WDDM TDR Delay</code>中的值，把
+<code>2</code>(默认一般是2)，调大，比如200。</li>
+<li>保存后退出，重启电脑即可。</li>
+</ol>
+<p>详见<a
+href="http://blog.163.com/yuhua_kui/blog/static/9679964420146183211348/">解决CUDA程序的黑屏恢复问题</a></p>
 ]]></content>
       <categories>
         <category>CUDA</category>
@@ -3197,6 +3200,48 @@ href="https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html">官方
 <p>矩阵乘法2.0</p>
 <p>使用 <code>__device__</code>获取矩阵中元素和设定元素</p>
 <figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// Matrices are stored in row-major order:</span></span><br><span class="line"><span class="comment">// M(row, col) = *(M.elements + row * M.stride + col)</span></span><br><span class="line"><span class="keyword">typedef</span> <span class="class"><span class="keyword">struct</span> &#123;</span></span><br><span class="line">    <span class="type">int</span> width;</span><br><span class="line">    <span class="type">int</span> height;</span><br><span class="line">    <span class="type">int</span> stride;</span><br><span class="line">    <span class="type">float</span>* elements;</span><br><span class="line">&#125; Matrix;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 获取矩阵元素</span></span><br><span class="line">__device__ <span class="type">float</span> <span class="title function_">GetElement</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">return</span> A.elements[row * A.stride + col];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 设定矩阵元素</span></span><br><span class="line">__device__ <span class="type">void</span> <span class="title function_">SetElement</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col,</span></span><br><span class="line"><span class="params">                           <span class="type">float</span> value)</span></span><br><span class="line">&#123;</span><br><span class="line">    A.elements[row * A.stride + col] = value;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 根据线程锁的大小构建给定矩阵的子矩阵</span></span><br><span class="line"> __device__ Matrix <span class="title function_">GetSubMatrix</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    Matrix Asub;</span><br><span class="line">    Asub.width    = BLOCK_SIZE;</span><br><span class="line">    Asub.height   = BLOCK_SIZE;</span><br><span class="line">    Asub.stride   = A.stride;</span><br><span class="line">    Asub.elements = &amp;A.elements[A.stride * BLOCK_SIZE * row</span><br><span class="line">                                         + BLOCK_SIZE * col];</span><br><span class="line">    <span class="keyword">return</span> Asub;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BLOCK_SIZE 16</span></span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(<span class="type">const</span> Matrix, <span class="type">const</span> Matrix, Matrix)</span>;</span><br><span class="line"></span><br><span class="line"><span class="type">void</span> <span class="title function_">MatMul</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">const</span> Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">    Matrix d_A;</span><br><span class="line">    d_A.width = d_A.stride = A.width; d_A.height = A.height;</span><br><span class="line">    <span class="type">size_t</span> size = A.width * A.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_A.elements, size);</span><br><span class="line">    cudaMemcpy(d_A.elements, A.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line">    Matrix d_B;</span><br><span class="line">    d_B.width = d_B.stride = B.width; d_B.height = B.height;</span><br><span class="line">    size = B.width * B.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_B.elements, size);</span><br><span class="line">    cudaMemcpy(d_B.elements, B.elements, size,</span><br><span class="line">    cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    Matrix d_C;</span><br><span class="line">    d_C.width = d_C.stride = C.width; d_C.height = C.height;</span><br><span class="line">    size = C.width * C.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_C.elements, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Invoke kernel</span></span><br><span class="line">    dim3 <span class="title function_">dimBlock</span><span class="params">(BLOCK_SIZE, BLOCK_SIZE)</span>;</span><br><span class="line">    dim3 <span class="title function_">dimGrid</span><span class="params">(B.width / dimBlock.x, A.height / dimBlock.y)</span>;</span><br><span class="line">    MatMulKernel&lt;&lt;&lt;dimGrid, dimBlock&gt;&gt;&gt;(d_A, d_B, d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Read C from device memory</span></span><br><span class="line">    cudaMemcpy(C.elements, d_C.elements, size,</span><br><span class="line">               cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Free device memory</span></span><br><span class="line">    cudaFree(d_A.elements);</span><br><span class="line">    cudaFree(d_B.elements);</span><br><span class="line">    cudaFree(d_C.elements);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiplication kernel called by MatMul()</span></span><br><span class="line"> __global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(Matrix A, Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 定义块的行与列</span></span><br><span class="line">    <span class="type">int</span> blockRow = blockIdx.y;</span><br><span class="line">    <span class="type">int</span> blockCol = blockIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程块计算每一个子矩阵</span></span><br><span class="line">    Matrix Csub = GetSubMatrix(C, blockRow, blockCol);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程计算每个矩阵的元素，结果保留在Cvalue中</span></span><br><span class="line">    <span class="type">float</span> Cvalue = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 定义Csub中线程的行列</span></span><br><span class="line">    <span class="type">int</span> row = threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 对矩阵A、B的所有子矩阵进行循环</span></span><br><span class="line">    <span class="comment">// 计算Csub，把每一对子矩阵Csub循环，之后累加求和</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> m = <span class="number">0</span>; m &lt; (A.width / BLOCK_SIZE); ++m) &#123;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 获取矩阵A、B的子矩阵</span></span><br><span class="line">        Matrix Asub = GetSubMatrix(A, blockRow, m);</span><br><span class="line">        Matrix Bsub = GetSubMatrix(B, m, blockCol);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 存储A、B矩阵的子矩阵Asub和Bsub用于共享</span></span><br><span class="line">        __shared__ <span class="type">float</span> As[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line">        __shared__ <span class="type">float</span> Bs[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 从设备中载入Asub与Bsub到共享内存中</span></span><br><span class="line">        <span class="comment">// 每个线程加载每个子矩阵的一个元素</span></span><br><span class="line">        As[row][col] = GetElement(Asub, row, col);</span><br><span class="line">        Bs[row][col] = GetElement(Bsub, row, col);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 在开始计算前同步以确保子矩阵被加载</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 同时计算A、B子矩阵</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> e = <span class="number">0</span>; e &lt; BLOCK_SIZE; ++e)</span><br><span class="line">            Cvalue += As[row][e] * Bs[e][col];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 同步以确保A、B子矩阵在下次迭代前计算完成</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 写入C子矩阵到设备内存中，每个线程写入一个元素</span></span><br><span class="line">    SetElement(Csub, row, col, Cvalue);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>CUDA</category>
+      </categories>
+      <tags>
+        <tag>GPU</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>CUDA编程入门</title>
+    <url>/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>CUDA编程入门，接 <a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a></p>
+<span id="more"></span>
+<p>参考文章：<a
+href="https://blog.csdn.net/augusdi/article/details/12833235">CUDA从入门到精通</a></p>
+<h2 id="注意事项">注意事项</h2>
+<p>远程桌面登陆服务器时，远程终端不能运行CUDA程序。原因是因为远程桌面登陆调用的显卡资源是本地的显卡，运行程序的时候没有远程的显卡，自然会报错。所以可以用远程终端进行登陆，或者远程服务器配置两块显卡，一块用于显示一块用于计算。</p>
+<p>话不多说，直接上代码。</p>
+<h2 id="原始的demo线程并行">原始的Demo（线程并行）</h2>
+<p>目的就是a, b两个数组对应元素进行相加</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 导入cuda相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="comment">// c中自带的模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 声明一个函数后面需要调用</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// __global__前缀表示用于控制设备，还有一个控制设备的前缀是__device__</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 这个函数是运行在GPU上的，称为核函数</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cuda_Error_t指的是cuda错误类型，取值为整数</span></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="comment">// cudaDeviceProp是设备属性结构体</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadExit must be called before exiting in order for profiling and</span></span><br><span class="line">    <span class="comment">// tracing tools such as Nsight and Visual Profiler to show complete traces.</span></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 用于使用CUDA并行添加向量的辅助函数</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// GPU设备端的数据指针</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// GPU状态指示</span></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 当电脑上有多个显卡支持CUDA时，选择某个GPU来进行运算。0表示设备号，还有1,2,3……</span></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 分配GPU设备端的内存。三个变量，所以需要分配三次。</span></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="comment">// 申请失败就报错</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝数据到GPU中，已有两个变量，所以需要拷贝两次</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 运行核函数。关键代码</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 同步线程</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    <span class="comment">// 释放GPU设备端内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>以上是线程并行，CUDA官方初始给的例子。</p>
+<p>启动核函数的调用过程，即<code>addKernel</code>函数。</p>
+<p>&lt;&lt;&lt;&gt;&gt;&gt;表示运行时配置符号，里面1表示只分配一个线程组（又称线程块、Block），size表示每个线程组有size个线程（Thread）。本程序中size根据前面传递参数个数应该为5，所以运行的时候，核函数在5个GPU线程单元上分别运行了一次，总共运行了5次。这5个线程是如何知道自己“身份”的？是靠<code>threadIdx</code>这个内置变量，它是个dim3类型变量，接受&lt;&lt;&lt;&gt;&gt;&gt;中第二个参数，它包含x,y,z
+3维坐标，而我们传入的参数只有一维，所以只有x值是有效的。通过核函数中<code>int i = threadIdx.x;</code>这一句，每个线程可以获得自身的id号，从而找到自己的任务去执行。</p>
+<h2 id="块并行">块并行</h2>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">	c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">addKernel&lt;&lt;&lt;size,<span class="number">1</span> &gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br></pre></td></tr></table></figure>
+<p>这里与线程并行的区别在于，由<code>threadIdx</code>改为了<code>blockIdx</code>。<code>&lt;&lt;&lt;1, size&gt;&gt;&gt;</code>改为了<code>&lt;&lt;&lt;size, 1&gt;&gt;&gt;</code>。</p>
+<p>线程并行是细粒度并行，调度效率高；块并行是粗粒度并行，每次调度都要重新分配资源，有时资源只有一份，那么所有线程块都只能排成一队，串行执行。</p>
+<p>有时采用分治法，将一个大问题分解为几个小规模问题，将这些小规模问题分别用一个线程块实现，线程块内可以采用细粒度的线程并行，而块之间为粗粒度并行，这样可以充分利用硬件资源，降低线程并行的计算复杂度。适当分解，降低规模，在一些矩阵乘法、向量内积计算应用中可以得到充分的展示。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/20130723220559500.png" /></p>
+<p>多个线程块组成一个线程格(Grid)。一维线程-&gt;二维线程块-&gt;三维线程格。</p>
+<h2 id="流并行">流并行</h2>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">addKernel&lt;&lt;&lt;size, <span class="number">1</span>&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">cudaStream_t stream[<span class="number">5</span>];</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">    <span class="comment">// 创建流</span></span><br><span class="line">    cudaStreamCreate(&amp;stream[i]);</span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">	<span class="comment">// 执行流。线程块数(1)，每个线程中线程数(1)，每个block用到的共享内存大小(0)，流对象(当前核函数在哪个流上运行)</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, <span class="number">1</span>, <span class="number">0</span>, stream[i]&gt;&gt;&gt;(dev_c + i, dev_a + i, dev_b + i);</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 控制整个设备的所有流同步</span></span><br><span class="line">cudaDeviceSynchronize();</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">Error:</span><br><span class="line">	<span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">        <span class="comment">//销毁流</span></span><br><span class="line">        cudaStreamDestroy(stream[i]);</span><br><span class="line">    &#125;</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br></pre></td></tr></table></figure>
+<p>利用多个网格（线程格）进行流并行。这样实现的并行可以达到比块并行更粗的细粒度。</p>
+<h2 id="线程通信">线程通信</h2>
+<p>这次目的分三个线程做三个任务，1-5数字进行求和，1-5每个数字求平方和，1-5累乘</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 更改addKernel函数内容如下</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a)</span> &#123;</span><br><span class="line">    <span class="comment">// 线程块数：1，块号为0</span></span><br><span class="line">    <span class="comment">// 线程数：5，线程号0~4</span></span><br><span class="line">    <span class="comment">// 共享存储器大小：5个int型变量大小（5 * sizeof(int）)</span></span><br><span class="line"></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 将数据一次读入到SM（Shared Memory）内部</span></span><br><span class="line">    <span class="keyword">extern</span> __shared__ <span class="type">int</span> smem[];</span><br><span class="line">    smem[i] = a[i];</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 0号线程做平方和</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">0</span>) &#123;</span><br><span class="line">        c[<span class="number">0</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">0</span>] += smem[d] * smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 1号线程做累加</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">1</span>) &#123;</span><br><span class="line">        c[<span class="number">1</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">1</span>] += smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 2号线程做累乘</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">2</span>) &#123;</span><br><span class="line">        c[<span class="number">2</span>] = <span class="number">1</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">2</span>] *= smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改main函数的内容</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line"> 	<span class="comment">// 跟普通的C语言代码编写差不多</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;\t1+2+3+4+5 = %d\n\t1^2+2^2+3^2+4^2+5^2 = %d\n\t1*2*3*4*5 = %d\n\n\n\n\n\n&quot;</span>, c[<span class="number">1</span>], c[<span class="number">0</span>], c[<span class="number">2</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改addWithCuda内容</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a,  <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 去除c变量相关内容，如申请内存、GPU设备端指针。其余内容差不多</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)&#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 第三个参数为共享内存大小（字节数）</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size, size * <span class="title function_">sizeof</span><span class="params">(<span class="type">int</span>)</span>, 0&gt;&gt;&gt;<span class="params">(dev_c, dev_a)</span>;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="性能剖析">性能剖析</h2>
+<p>测试对比线程并行和块并行的速度</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_blk</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_thd</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">1024</span>;</span><br><span class="line">    <span class="type">int</span> a[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="type">int</span> b[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; arraySize;i++) &#123;</span><br><span class="line">        a[i] = i;</span><br><span class="line">        b[i] = arraySize-i;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line">    <span class="type">int</span> num = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// 设备属性结构体</span></span><br><span class="line">    cudaDeviceProp prop;</span><br><span class="line">    <span class="comment">// 获取设备总数</span></span><br><span class="line">    cudaStatus = cudaGetDeviceCount(&amp;num);</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;num;i++) &#123;</span><br><span class="line">        <span class="comment">// 获取设备属性</span></span><br><span class="line">        cudaGetDeviceProperties(&amp;prop,i);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;arraySize;i++) &#123;</span><br><span class="line">        <span class="keyword">if</span>(c[i] != (a[i]+b[i])) &#123;</span><br><span class="line">            <span class="built_in">printf</span>(<span class="string">&quot;Error in %d\n&quot;</span>,i);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span> &#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy input vectors from host memory to GPU buffers.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 计算核函数执行时间</span></span><br><span class="line">    cudaEvent_t start, stop;</span><br><span class="line">    cudaEventCreate(&amp;start);</span><br><span class="line">    cudaEventCreate(&amp;stop);</span><br><span class="line">    cudaEventRecord(start, <span class="number">0</span>);</span><br><span class="line">    <span class="comment">// 由于数据量小，执行1000次扩大执行时间。blk是块并行，thd是线程并行</span></span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">1000</span>;i++) &#123;</span><br><span class="line"><span class="comment">//      addKernel_blk&lt;&lt;&lt;size,1&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span></span><br><span class="line">        addKernel_thd&lt;&lt;&lt;<span class="number">1</span>,size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaEventRecord(stop,<span class="number">0</span>);</span><br><span class="line">    cudaEventSynchronize(stop);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// stop-start，执行总时间</span></span><br><span class="line">    <span class="type">float</span> tm;</span><br><span class="line">    cudaEventElapsedTime(&amp;tm, start, stop);</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;GPU Elapsed time:%.6f ms.\n&quot;</span>, tm);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadSynchronize waits for the kernel to finish, and returns</span></span><br><span class="line">    <span class="comment">// any errors encountered during the launch.</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy output vector from GPU buffer to host memory.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
         <category>CUDA</category>
@@ -3313,4640 +3358,4595 @@ href="https://blog.csdn.net/xuxiatian/article/details/50577960">这篇文章</a>
       </tags>
   </entry>
   <entry>
-    <title>CUDA编程二次入门</title>
-    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>本文主要介绍了基本的CUDA程序框架及代码解析，包括如何申请设备指针内存、将数据拷贝到设备上、调用核函数进行计算、线程同步以及将结果拷贝回主机内等。同时也介绍了一些常见问题及解决方案，比如CUDA程序黑屏之后恢复的问题。接
-<a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a> 。阅读本文前需要有一部分C语言基础。</p>
-<span id="more"></span>
-<h2 id="原始代码">原始代码</h2>
-<p>在VS2015中，新建一个CUDA项目，里面会有一个
-<code>kernel.cu</code>的文件。里面的代码就是官方给出的实例代码。如下所示。</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h2 id="拆解原始代码">拆解原始代码</h2>
-<p>下面对上面的代码逐一进行拆解进行说明。最原始最简单的代码应该是下面这样。在配置正确的情况下，下面的代码应该也是可以正确运行并输出的。</p>
-<p>设备指的是GPU设备</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 引入相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 调用核函数，利用gpu进行计算</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="comment">// 获取当前线程id</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 在同一个线程id下，a数组与b数组元素对应相加</span></span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="comment">// 一般c语言的语法，在这里不做介绍</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针，每个变量对应一个设备指针。所以这里有三个</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 选择GPU设备，如果只有一个就只需选择0</span></span><br><span class="line">    cudaSetDevice(<span class="number">0</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针的内存</span></span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将主机上的a、b数组的数据拷贝到设备上。</span></span><br><span class="line">    cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用核函数，注意这里调用函数的语法与一般的不同。&lt;&lt;&lt;1, size&gt;&gt;&gt;指的是分配一个线程组，里面有size个线程</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 进行线程同步</span></span><br><span class="line">    cudaThreadSynchronize();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机里的c变量中</span></span><br><span class="line">    cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备指针内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 输出结果</span></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>, c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 退出线程</span></span><br><span class="line">    cudaThreadExit();</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p><code>kernel.cu</code>里面的代码是基于上面这个代码进行拓展。</p>
-<ol type="1">
-<li>每一次操作之后，进行判断状态
-<code>cudaStatus</code>，如果状态异常就退出，这是一种比较好的编程习惯，当然带来的影响就是可读性略微降低了。</li>
-<li>封装了一个函数，将操作主机的代码与操作设备的代码（比如申请设备指针内存与释放设备指针等）。</li>
-</ol>
-<h2 id="拓展代码">拓展代码</h2>
-<p>当然，我们还可以基于此再次进行拓展。下面介绍几个比较常用的拓展代码。</p>
-<p>待补充。。。。。</p>
-<h2 id="补充说明">补充说明</h2>
-<p>如果在运行cuda代码的时候，显示屏突然黑屏，之后又恢复正常，是因为程序时间运行过长，触发了TDR事件，导致黑屏，显卡运算中断。解决方案如下：</p>
-<ol type="1">
-<li>开始菜单中找到
-<code>Nsight Monitor</code>并打开。（Win10可以直接搜索）</li>
-<li>在任务栏中打开 <code>Nsight Monitor</code>，单击右下角的
-<code>Nsight Monitor options</code>。</li>
-<li>设置 <code>General</code>的 <code>Microsoft Display Driver</code>中
-<code>WDDM TDR Delay</code>中的值，把
-<code>2</code>(默认一般是2)，调大，比如200。</li>
-<li>保存后退出，重启电脑即可。</li>
-</ol>
-<p>详见<a
-href="http://blog.163.com/yuhua_kui/blog/static/9679964420146183211348/">解决CUDA程序的黑屏恢复问题</a></p>
-]]></content>
-      <categories>
-        <category>CUDA</category>
-      </categories>
-      <tags>
-        <tag>GPU</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>用 Cpp 写 PyTorch 的插件</title>
-    <url>/hexo_blog/2023/12/03/PyTorch/pytorch_cpp_extension/</url>
-    <content><![CDATA[<p>从零开始，用 Cpp 写 PyTorch 的插件，包括 CPU 和 GPU 的版本。</p>
-<p><a
-href="https://github.com/wnma3mz/pytorch_cuda_extension">代码</a></p>
+    <title>OpenWrt路由器折腾记录</title>
+    <url>/hexo_blog/2023/01/10/IOT/OpenWrt%E8%B7%AF%E7%94%B1%E5%99%A8%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>在路由器上安装和配置OpenWrt。它包括安装软件包和设置防火墙、接口和各种网络连接的分步说明。</p>
 <span id="more"></span>
-<h2 id="为什么">为什么</h2>
-<p>一般来说，在原生功能不能满足需求的时候，插件可以作为补充。比如，PyTorch
-的 <code>torch.nn.functional</code> 中没有 <code>softmax</code>
-函数，但是 <code>torch.nn</code> 中有，所以可以用
-<code>torch.nn.functional.softmax</code> 来代替。但是，如果要用
-<code>softmax</code> 的导数，就需要用到 <code>softmax</code>
-的原始定义，这个时候就需要自己写插件了。</p>
-<p>如果是比较简单的需求，则可以直接用 Python
-完成。然而，当对性能要求较高时，往往会使用 Cpp
-来写插件，最后甚至会优化为 CUDA 代码。</p>
-<h2 id="怎么写">怎么写</h2>
-<p>从例子出发，一步步来写。假设要实现一个最简单的 Attention 模块，输入为
-<span class="math inline">\(q,k,v \in \mathbb{R}^{M \times
-N}\)</span>，输出为 <span class="math inline">\(out \in \mathbb{R}^{M
-\times N}\)</span>（不考虑 Batch Size 以及 Head 数量的情况）。Attention
-模块的计算公式为：</p>
-<p><span class="math display">\[
-out = \text{softmax}(qk^T)v
-\]</span></p>
-<p>只实现 <code>forward</code> 函数，不实现 <code>backward</code>
-函数。</p>
-<h3 id="cpu-版本">CPU 版本</h3>
-<p>类似于写 Python 的库，创建一个文件夹，目录结构如下所示：</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">attention</span><br><span class="line">├── attention.cpp</span><br><span class="line">├── setup.py</span><br><span class="line">└── __init__.py</span><br></pre></td></tr></table></figure>
-<h4 id="python-部分">Python 部分</h4>
-<p>核心代码在 <code>attention.cpp</code> 中 ，首先在
-<code>setup.py</code> 中添加如下代码：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CppExtension</span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CppExtension(<span class="string">&#x27;attention&#x27;</span>, [<span class="string">&#x27;attention.cpp&#x27;</span>]),</span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
-<p>这样，就可以用 <code>python setup.py install</code>
-来安装插件了。或者用 <code>pip install -e attention</code>
-以便于快速调试</p>
-<p>在 <code>__init__.py</code> 中导入 <code>attention</code> 模块，以在
-Python 中调用 <code>forward</code> 函数，直接计算 attention。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> .attention <span class="keyword">import</span> forward</span><br></pre></td></tr></table></figure>
-<h4 id="cpp-部分">Cpp 部分</h4>
-<p>接下来，我们需要在 <code>attention.cpp</code> 中实现
-<code>forward</code> 函数，为进行区分，这里使用这个函数名称
-<code>attention_forward</code> ，这个函数的输入是 q、k、v 三个
-<code>Tensor</code>，输出是
-<code>torch::Tensor</code>。而具体的计算步骤可以拆解为三个步骤：</p>
-<ol type="1">
-<li>矩阵的乘法</li>
-<li>softmax</li>
-<li>矩阵的乘法</li>
-</ol>
-<p>使用 <code>PYBIND11_MODULE</code>把
-<code>attention_forward</code>函数暴露出去，绑定到
-<code>forward</code>上，这样就能用 <code>forward</code>函数来调用
-<code>attention_forward</code>。整合后的完整代码如下，</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="comment">// 返回：方便起见，返回一个 vector，实际上只有一个元素，便于后续扩展</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = torch::<span class="built_in">matmul</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>));</span><br><span class="line">    scores = torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>);</span><br><span class="line">    torch::Tensor attention = torch::<span class="built_in">matmul</span>(scores, v);</span><br><span class="line">    <span class="keyword">return</span> &#123;attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CPU)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="测试">测试</h4>
-<p>用 Python 版本的实现来测试 Cpp 版本的实现是否正确，测试代码如下，</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> attention</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">torch.manual_seed(<span class="number">42</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_attention</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>) @ v</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_attention(q, k, v)</span><br><span class="line">    cpp_values = attention.forward(q, k, v)[-<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">q, k, v, loop=<span class="number">100</span></span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_attention(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: attention.forward(q, k, v), number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span> <span class="keyword">if</span> torch.cuda.is_available() <span class="keyword">else</span> <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;q&quot;</span>, q)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;k&quot;</span>, k)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;v&quot;</span>, v)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">20</span>)</span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    compare_time(q, k, v)</span><br></pre></td></tr></table></figure>
-<p>测试通过后，可以再使用 <code>compare_time</code>
-函数对比一下二者的速度。理论上，二者的速度是相差无几的。因为均用的是
-PyTorch 的矩阵乘法和 softmax 函数。</p>
-<p>但是，如果需要进行更进一步的优化技巧，那么就需要自己实现矩阵乘法和
-softmax 函数了。这里，我们只实现最简单的矩阵乘法和 softmax
-函数，然后再对比一下二者的速度。</p>
-<h4 id="矩阵乘法">矩阵乘法</h4>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="softmax">softmax</h4>
-<p>由于
-<code>softmax</code>函数比较特殊，后续会结合算子融合一起优化，所以简单的对其进行展开。用
-<code>torch::exp</code>和
-<code>torch::sum</code>实现了一遍，为了方便也可以直接使用
-<code>torch::softmax(scores, 1)</code></p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_softmax</span><span class="params">(<span class="type">const</span> torch::Tensor&amp; scores)</span> </span>&#123;</span><br><span class="line">    torch::Tensor exponents = torch::<span class="built_in">exp</span>(scores);</span><br><span class="line">    torch::Tensor sum = torch::<span class="built_in">sum</span>(exponents, <span class="number">1</span>, <span class="literal">true</span>);</span><br><span class="line">    <span class="keyword">return</span> exponents / sum;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>但把这两个函数替换到 <code>attention_forward</code>函数中后，再次运行
-<code>compare_time</code>函数，发现手写的 Cpp 版本的实现要比 Python
-版本的实现慢。为什么？因为，当前只是简单的实现了矩阵乘法和 softmax，而
-PyTorch 中的矩阵乘法和 softmax 都是经过优化的，所以速度会更快。</p>
-<p>另外，使用原生的矩阵乘法和 softmax 函数，可以在 GPU
-上运行，而手写的矩阵乘法和 softmax 函数，只能在 CPU
-上运行。因此，接下来将其改造为 GPU 版本，然后再进行优化。</p>
-<h3 id="gpu-版本">GPU 版本</h3>
-<h4 id="python-部分-1">Python 部分</h4>
-<p>在 <code>setup.py</code> 中更改为如下代码，把
-<code>CppExtension</code> 改为 <code>CUDAExtension</code></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CUDAExtension </span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CUDAExtension(<span class="string">&#x27;attention&#x27;</span>, [</span><br><span class="line">            <span class="string">&#x27;attention.cpp&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;attention_kernel.cu&#x27;</span>,</span><br><span class="line">        ])      </span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="cpp-部分-1">Cpp 部分</h4>
-<p>为了兼容之前的代码，这里将之前的 <code>attention_forward</code>更新为
-<code>attention_cpu_forward</code>，同时加了一个类型判断，如果输入的
-<code>Tensor</code>不在同一个设备上，则抛出异常。而对于
-<code>attention_cuda_forward</code>的实现需要在
-<code>attention_kernel.cu</code>中实现。注意：这里需要提前定义好
-<code>attention_cuda_forward</code>函数，否则会报错。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span></span>;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cpu_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line">    torch::Tensor scores = <span class="built_in">my_matmul</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">my_matmul</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;v)</span> </span>&#123;</span><br><span class="line">    <span class="keyword">if</span> (!(q.<span class="built_in">device</span>().<span class="built_in">type</span>() == k.<span class="built_in">device</span>().<span class="built_in">type</span>() &amp;&amp; q.<span class="built_in">device</span>().<span class="built_in">type</span>() == v.<span class="built_in">device</span>().<span class="built_in">type</span>())) &#123;</span><br><span class="line">        <span class="keyword">throw</span> std::<span class="built_in">runtime_error</span>(<span class="string">&quot;Input tensors q, k, and v must be on the same device&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (q.<span class="built_in">is_cuda</span>()) &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cuda_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cpu_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CUDA)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="cu-部分">Cu 部分</h4>
-<p>在同级目录下，创建 <code>attention_kernel.cu</code>文件。首先实现
-<code>attention_cuda_forward</code>函数的主要逻辑。其中，主要对矩阵乘法进行了优化，使用了
-CUDA 的并行计算。然后，使用
-<code>AT_DISPATCH_FLOATING_TYPES</code>宏来实现对不同类型的支持，这样就可以支持
-<code>float</code>和 <code>double</code>类型了。</p>
-<p>对于
-<code>matrix_multiply</code>函数，与一般写法不同的是，需要提前创建好输出的
-<code>Tensor</code>，然后再传入到 CUDA 的函数中。并且需要创建好
-<code>blocks</code>和 <code>threads</code>，然后再调用 CUDA
-的函数。这里指定了每个 CUDA 是有 16 x 16
-线程的块，而这些块是可以并行计算的，所以能够加速计算。可参考 <a
-href="https://devblogs.nvidia.com/even-easier-introduction-cuda">An Even
-Easier Introduction to CUDA</a></p>
-<p>而在传递参数的时候，需要使用 <code>packed_accessor</code>。这里的
-<code>packed_accessor</code>的第一个参数是
-<code>Tensor</code>的类型，第二个参数是
-<code>Tensor</code>的维度，第三个参数是
-<code>Tensor</code>的类型，第四个参数是
-<code>Tensor</code>的维度。这里的
-<code>packed_accessor</code>的第三个参数和第四个参数，是为了支持 CUDA
-的。</p>
-<p>接下来就是实现
-<code>matrix_multiply_kernel</code>。矩阵的乘法中，如果要计算输出矩阵的第一个值，则需要用到输入矩阵的第一行和第一列。因此，这里需要根据
-<code>block</code>和
-<code>thread</code>的索引，来计算出对应的行和列。然后，就是普通的矩阵乘法的实现了。原来的矩阵乘法的实现是：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">out = [[<span class="number">0</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(m)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(m):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(p):</span><br><span class="line">            out[i][j] += input1[i][k] * input2[k][j]</span><br></pre></td></tr></table></figure>
-<p>相当于把外面两个循环分别交给了 CUDA 的 <code>block</code>和
-<code>thread</code>来计算。这样，就可以实现并行计算了。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row][col] = value;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">matrix_multiply</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>测试代码不变，依旧可以用上面的来进行检验。至此，最简单的实验就完成了。接下来，就是对其进行优化了。</p>
-<h2 id="矩阵乘法的优化">矩阵乘法的优化</h2>
-<h3 id="matmul">Matmul</h3>
-<p>重新回到矩阵乘法上，假设有两个矩阵 A1 和 A2，形状分别为 <span
-class="math inline">\(M \times N\)</span> 和 <span
-class="math inline">\(N \times K\)</span>，则矩阵乘法的计算公式为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">output = torch.zeros(size=(M, K))</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        sum_ = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(N):</span><br><span class="line">            sum_ += A1[i][k] * A2[k][j]</span><br><span class="line">        output[i][j] = sum_</span><br></pre></td></tr></table></figure>
-<p>一种朴素的优化手段是把最后一个循环并行计算。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        output[i][j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br></pre></td></tr></table></figure>
-<p>利用多线程/进程（下文统称为job）进行并行计算可以提高程序的计算速度，但这样需要每个job都能访问到
-A1 和 A2 的数据，所以这就引入了全局内存和共享内存的概念。</p>
-<blockquote>
+<h2 id="基本界面">基本界面</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353557758.png" /></p>
+<p>准备：</p>
 <ul>
-<li>全局内存（Global
-Memory）：全局内存是一种在计算机程序中可被所有线程或进程访问的内存空间。它通常用于存储全局变量、静态变量以及动态分配的内存等。全局内存的特点是可以在整个程序执行过程中进行读写操作，但它的访问速度相对较慢。</li>
-<li>共享内存（Shared
-Memory）：共享内存是一种特殊的内存区域，被多个线程或进程同时访问和共享。通过将数据存储在共享内存中，不同的线程或进程可以直接读取和写入这些数据，而无需使用其他的通信机制。共享内存的特点是高效的数据共享和访问速度，因为不需要进行复制或传输数据。</li>
+<li>固件</li>
+<li>待恢复的配置</li>
+<li>需要安装的软件+运行的命令</li>
 </ul>
-</blockquote>
-<p>CUDA 中依旧存在类似的概念</p>
-<blockquote>
+<p>配置出问题，直接重刷。同时方便起见，要及时备份当前的配置。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353799884.png" /></p>
+<h2 id="软件包的安装">软件包的安装</h2>
+<p>能在线安装的可以直接在线安装。直接从"系统-&gt;软件包"里面找到需要安装的软件。</p>
+<p>但有时候在线安装会由于网络，软件源、软件版本等问题，无法自动安装。因此需要我们手动找到ipk文件进行安装。使用下面的命令进行安装。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">opkg install XXX</span><br></pre></td></tr></table></figure>
+<p>官方软件下载网页：<a
+href="https://downloads.openwrt.org/releases/">https://downloads.openwrt.org/releases/</a></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 系统版本可以直接从管理界面的概况中得知，这里是17.01.4</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 内核版本</span></span><br><span class="line">&gt;&gt;&gt; <span class="built_in">uname</span> -a </span><br><span class="line">Linux dw33d 4.4.194 <span class="comment">#0 Tue Oct 17 17:46:20 2017 mips GNU/Linux</span></span><br><span class="line"><span class="comment"># 查看支持的架构</span></span><br><span class="line">&gt;&gt;&gt; opkg print-architecture</span><br><span class="line"><span class="built_in">arch</span> all 1</span><br><span class="line"><span class="built_in">arch</span> noarch 1</span><br><span class="line"><span class="built_in">arch</span> mips_24kc 10</span><br></pre></td></tr></table></figure>
+<p>对应我这里的第三方包就是：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/</a></p>
+<p>基础的库：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base</a></p>
+<p>luci：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/</a></p>
+<h3 id="tmux">tmux</h3>
+<p>以tmux安装为例，该软件无法直接安装。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 1. 从package里面下载tmux的ipkg文件</span></span><br><span class="line"><span class="comment"># 2. 执行安装命令</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 3. 报错信息，一些依赖文件没有。所以从base里面找到对应缺失的ipk文件。并进行安装</span></span><br><span class="line">&gt;&gt;&gt;opkg install terminfo_6.0-1_mips_24kc.ipk libncurses_6.0-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 4. 此时，再安装tmux。安装成功</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 但安装libevent2这个库时，提示当前已安装，并且版本过高，并不会自动降级。所以需要手动卸载原有的软件</span></span><br><span class="line">&gt;&gt;&gt;opkg remove libevent2 --force-depends</span><br><span class="line"><span class="comment"># 卸载失败，提示已有软件依赖该库。所以我们需要强制进行卸载，将依赖软件也进行卸载。加上--force 命令</span></span><br><span class="line"><span class="comment"># 之后再安装低版本的libncurses</span></span><br><span class="line">&gt;&gt;&gt;opkg install libevent2_2.0.22-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后再运行tmux，成功</span></span><br></pre></td></tr></table></figure>
+<p>其他软件应该同理，需要一步一步找到依赖文件，并且耐心地解决各种问题。dw33d内置了一个16g的sd卡，所以我把一些重装必备的ipk文件放到里面，便于快速安装。并且刷写固件并不会影响该sd卡的内容，可以放心存储。</p>
+<h3 id="tailscale">Tailscale</h3>
+<p>异地组网工具，从官网下载找到mips版本，放到sd卡中，避免刷新固件有影响。相当于单文件就可以运行。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压文件</span></span><br><span class="line">&gt;&gt;&gt; tar zxvf tailscale_1.34.1_mips.tgz</span><br><span class="line"><span class="comment"># 到对应的目录中，里面会有一些文件。主要是两个可执行文件tailscale和tailscaled。</span></span><br><span class="line"><span class="comment"># 第一次运行需要启动tailscaled服务</span></span><br><span class="line">&gt;&gt;&gt;./tailscaled --state=tailscaled.state</span><br><span class="line"><span class="comment"># 此时需要新开一个窗口，登录tailscale</span></span><br><span class="line">&gt;&gt;&gt;./tailscale login </span><br><span class="line"><span class="comment"># 可以直接把网址复制出来，用其他设备的浏览器进行登录</span></span><br></pre></td></tr></table></figure>
+<p>登录完毕之后，组网就完成了。此时，可以关闭该窗口。但为了后台启动，所以可以使用tmux，或者nohup命令。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
+<p>同时为了保证开机自启动，所以我们可以在web管理界面中"系统-&gt;启动项"中，最后的"本地启动脚本"中插入(<code>exit 0</code>之前行)</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先切换到对应目录，再执行</span></span><br><span class="line"><span class="built_in">cd</span> /mnt/sda1/tailscale_1.34.1_mips &amp;&amp; <span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
+<p>这样，就能保证异地管理该路由器的web界面，并且可以直接ssh登录</p>
+<h2 id="防火墙">防火墙</h2>
+<p>"网络-&gt;防火墙"中的默认配置，wan口是拒绝入站数据的。这样就会导致如果该路由器跟其他设备同属于一个局域网，ssh和web界面均不能通过ip登录。具体的例子是：</p>
 <ul>
-<li>全局内存（Global Memory）：在 CUDA
-中，全局内存是一个设备（GPU）上可见的主机（CPU）内存空间。它可以由所有的线程块和线程访问，用于存储全局变量和动态分配的内存等。全局内存的读写操作相对较慢，因为涉及主机与设备之间的数据传输。</li>
-<li>共享内存（Shared Memory）：在 CUDA
-中，共享内存是位于每个线程块中的一块高速缓存内存。它被同一个线程块内的线程共享，并且比全局内存具有更快的读写速度。共享内存通常用于优化算法的性能，通过在线程块内部共享数据来减少全局内存的访问。</li>
+<li>该路由器连接了一个上级路由器A，获取到的IP是192.168.0.X。分配给下面的设备的IP是192.168.1.X</li>
+<li>另一个设备接入路由器A时，获取到的IP是192.168.0.Y。此时能ping通192.168.0.X，但是无法进行ssh，也无法通过192.168.0.X访问管理界面</li>
+<li>另一个设备接入该路由器时，获取到的IP是192.168.1.Y。此时能够通过ssh和web访问192.168.1.X。</li>
 </ul>
-</blockquote>
-<p>简而言之，全局内存可以很方便的存各种东西，但是速度慢；共享内存是一个好东西，速度块，但通常大小受限。所以，分别有两种优化：</p>
+<p>需求：在同局域网下的其他设备也能访问该路由器。</p>
+<p>做法：</p>
 <ul>
-<li>全局内存中，考虑如何加速访问</li>
-<li>共享内存中，考虑如何减小占用空间</li>
+<li>一个简单的做法就是如下图所示，把"防火墙"中的"常规设置"里面的wan口，入站数据从<strong>拒绝</strong>改成<strong>接受</strong>。（默认是拒绝入站的）</li>
 </ul>
-<p>这就引入了 Tiled matmul 算法。</p>
-<h3 id="tiled-matmul">Tiled Matmul</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1680509223.png" /></p>
 <ul>
-<li>对于加速访问，在无法控制硬件的前提下，只能通过并行的方式同时读取数据。</li>
-<li>对于减小占用空间，可以通过拆分矩阵，把大矩阵拆分成若干小矩阵，然后再进行计算。</li>
+<li>另一个做法是配置"防火墙-&gt;端口转发"，把需要配置的端口转发出来。如下图所示，web界面的端口是80。所以从wan口出来的访问80端口的流量，直接转发到192.168.4.1（该路由器分配给下面的IP，不是获取的IP）</li>
 </ul>
-<p>重新思考矩阵
-<code>output</code>的计算过程，每个元素的计算其实是独立的，其本质可以拆成若干独立的小块，如下图所示：</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/setup-2d0f22ecd2e9b7c84af56792d14ba18a.gif?raw=true" />
-From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
-<p>由于矩阵
-<code>output</code>每个元素是完全独立的，可以将其拆成若干个小矩阵来计算。如上图所示，把矩阵
-<code>output</code> 拆成了 4 个小矩阵。对应的代码如下所示：</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356610477.png" /></p>
+<p>关于流量规则这里，则是在系统内部决定是否开放某个端口。可以参考默认的一些配置进行开放，一般应该是不需要调整的。</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/block_matrix.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size = <span class="number">2</span> <span class="comment"># 方便起见，这里设置为 N,M,K 的公约数。同时也拆成了 2x2 的 block</span></span><br><span class="line">block_M, block_N, block_K = M // block_size, N // block_size, K // block_size</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line">output11 = matmul(A1[:block_M, :], A2[:, :block_K])</span><br><span class="line">output12 = matmul(A1[:block_M, :], A2[:, block_K:])</span><br><span class="line">output21 = matmul(A1[block_M:, :], A2[:, :block_K])</span><br><span class="line">output22 = matmul(A1[block_M:, :], A2[:, block_K:])</span><br><span class="line">output = torch.cat([torch.cat([output11, output12], dim=<span class="number">1</span>), torch.cat([output21, output22], dim=<span class="number">1</span>)], dim=<span class="number">0</span>)</span><br><span class="line"><span class="built_in">print</span>(output)</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(output, A1 @ A2)</span><br></pre></td></tr></table></figure>
-<p>对于，左上角矩阵 <code>output11</code>，实际上是由
-<code>block_size</code>个矩阵乘法，再求和得到的
-<code>output11 = matmul(A1[:block_M, :block_N], A2[:block_N, :block_K]) + matmul(A1[:block_M, block_N:], A2[block_N:, :block_K])</code>。再把它扩展的灵活一点</p>
-<ol type="1">
-<li>不局限于只能扩展为 2 <span class="math inline">\(\times\)</span> 2
-矩阵</li>
-<li>block_size 可以针对 M, N, K 进行调整</li>
-</ol>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                <span class="comment"># 计算每个 block 的矩阵乘法</span></span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                <span class="comment"># 把每个 block 的结果放到对应的位置</span></span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"><span class="built_in">print</span>(matmul(A1, A2))</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(matmul(A1, A2), A1 @ A2)</span><br></pre></td></tr></table></figure>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356804612.png" /></p>
+<h2 id="接口">接口</h2>
+<p>"网络-&gt;接口"</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/tmm-59dd890f48435e692c47919d0df4a5e6.gif" />
-From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
-<p>再次回顾这样做的目的，<a
-href="https://penny-xu.github.io/blog/tiled-matrix-multiplication">原作者</a>是这么说的：</p>
-<blockquote>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357070459.png" /></p>
+<p>我主要会有以下四种上网方式（不会专业术语，简单描述现象）：</p>
 <ul>
-<li>With or without tiling, the same number of accesses into global
-memory occur. The difference is that, without tiling, each thread must
-sequentially (one after the other) access global memory 8 times.</li>
-<li>With tiling, we can parallelize the access to global memory so that
-each thread only sequentially accesses global memory 4 times.</li>
-<li>To summarize, the point is not to reduce the number of
-multiplications or even the total number of global memory accesses, but
-rather to reduce the number of sequential global memory accesses per
-thread. In other words, we better share the heavy load of memory access
-across threads.</li>
+<li><p>桥接光猫</p>
+<ul>
+<li>将路由器WAN口接入光猫LAN口，代替光猫进行拨号，需要知道宽带的账号密码。</li>
+<li>在WAN口中，协议选择<strong>PPPoE</strong>，输入账号密码</li>
+<li>此时该路由器就是主路由器，一般同时有ipv4和ipv6两个地址，假设ipv4的ip为192.168.1.1</li>
+<li>接入该路由器的设备，由该路由器进行分配ip，ip与路由器同网段，如192.168.1.2</li>
+</ul></li>
+<li><p>桥接上一级路由器</p>
+<ul>
+<li>将路由器WAN口接入上一级路由器LAN口，无需知道账号密码</li>
+<li>在WAN口中，协议选择<strong>DHCP客户端</strong></li>
+<li>在物理设置中，选择桥接接口，默认<strong>以太网交换机:
+eth0</strong>开启。勾选两个无线网络。如下图所示</li>
+<li><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673524264094.png" /></li>
+<li>此时，IP同上一级路由器同网段，且如果有IPv6，则连接该路由器的设备也有IPv6</li>
+</ul></li>
+<li><p>做NAT</p>
+<ul>
+<li>将路由器WAN接入上一级路由器的LAN口，相当于把该路由器做一个新的入口。</li>
+<li>在WAN中，协议选择<strong>DHCP客户端</strong>或<strong>静态地址</strong>（需要额外配置）</li>
+<li>此时，接入该路由器的设备会独立上一级路由器的设备，即网段不同。</li>
+<li>注：如果此时还需要ipv6，可以参考这篇文件进行配置。<a
+href="https://www.lategege.com/?p=676">https://www.lategege.com/?p=676</a>
+<ul>
+<li>简单来说，就是需要选择静态IP，并在<strong>WAN口</strong>和<strong>LAN</strong>中的"DHCP服务器"中的IPv6设置中，路由器广告服务，DHCPv6服务，NDP-代理三个均选择中继模式</li>
+<li>并且在终端中 <code>vim /etc/config/dhcp</code>，找到
+<code>config dhcp 'wan'</code>对应的选项，加入
+<code>option master '1'</code>这一行</li>
+</ul></li>
+</ul></li>
+<li><p>WiFi放大器</p>
+<ul>
+<li><p>无需网线</p></li>
+<li><p>"网络-&gt;无线"中的搜索，找到需要放大WiFi，加入网络。看情况选择放大2.4G还是5G的WiFi</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357519480.png" /></p></li>
+<li><p>WPA密钥就是WiFi密码，新的网络名称可以默认也可以另起一个以便区分。</p></li>
+<li><p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357576678.png" /></p></li>
+<li><p>此时接口中会多一个接口</p></li>
+<li><p>其他同桥接</p></li>
+</ul></li>
 </ul>
-</blockquote>
-<p>简而言之，作者的观点是通过拆成 block
-的形式，能够并行的读取全局内存数据。个人观点，在某些情况下，拆分后的
-block 可以恰好把数据放到共享内存中，以加速计算？</p>
-<p>最后，对于 <code>block size</code>大小的选择：</p>
+]]></content>
+      <categories>
+        <category>Linux</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>路由器</tag>
+        <tag>OpenWrt</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>安卓手机利用Termux安装Linux</title>
+    <url>/hexo_blog/2021/07/21/IOT/%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E5%88%A9%E7%94%A8Termux%E5%AE%89%E8%A3%85Linux/</url>
+    <content><![CDATA[<p>本文介绍如何在安卓手机上安装和运行Ubuntu系统，并通过ssh连接进行操作，同时给出了一些其他有用的命令和配置说明。</p>
+<span id="more"></span>
+<h3 id="安装">安装</h3>
+<p>不要从Google Play上安装，正确的安装包地址<a
+href="https://f-droid.org/packages/com.termux/">f-droid</a></p>
+<p>下载完成后，正常安装，但需要额外的给存储空间权限。</p>
+<p>虽然说可以免root安装，但没有root权限，有些操作不能很好的完成，如
+<code>top</code></p>
+<h3 id="连接">连接</h3>
+<p>在手机上配置太麻烦，所以可以使用同一局域网下的电脑进行配置</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 手机上</span></span><br><span class="line"><span class="comment"># 安装ssh服务</span></span><br><span class="line">pkg install openssh -y</span><br><span class="line"><span class="comment"># 开启ssh服务器</span></span><br><span class="line">sshd</span><br><span class="line"><span class="comment"># 配置密码</span></span><br><span class="line">passwd</span><br><span class="line"><span class="comment"># 输入想要配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 电脑终端, ip填写对应的手机ip，8022是termux默认的ssh端口</span></span><br><span class="line">ssh root@[ip] -p8022</span><br><span class="line"><span class="comment"># 输入刚刚的密码</span></span><br></pre></td></tr></table></figure>
+<p>至此，就能在电脑端进行配置</p>
+<h3 id="配置">配置</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装第三方库</span></span><br><span class="line">pkg install proot git python wget -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># 一键安装项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/sqlsec/termux-install-linux.git</span><br><span class="line"><span class="built_in">cd</span> termux-install-linux</span><br><span class="line">python termux-linux-install.py</span><br><span class="line"><span class="comment"># 选择Ubuntu</span></span><br><span class="line"><span class="built_in">cd</span> ~/Termux-Linux/Ubuntu</span><br><span class="line">./start-ubuntu.sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或者，ubuntu版本，最新支持20.04</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/MFDGaming/ubuntu-in-termux</span><br><span class="line"><span class="built_in">cd</span> ubuntu-in-termux</span><br><span class="line"><span class="built_in">chmod</span> +x ubuntu.sh</span><br><span class="line">./ubuntu.sh -y</span><br><span class="line">./startubuntu.sh</span><br></pre></td></tr></table></figure>
+<p>这样安装的Linux是与Termux共享文件目录的，所以可以直接将文件拷贝在
+<code>ubuntu-fs/root</code>文件夹中，启动
+<code>./startubuntu.sh</code>后，就能直接看到文件。</p>
+<p>注：若进入系统后，无法更新软件源，可能是dns存在问题，可以在termux中更改dns解析（因为系统中不好编辑）。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim ubuntu-fs/etc/resolv.conf</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">nameserver 114.114.114.114</span><br><span class="line">nameserver 8.8.8.8</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 若依旧不行，可尝试其他dns，如1.1.1.1等</span></span><br></pre></td></tr></table></figure>
+<h3 id="其他">其他</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pkg install vim tmux -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># frp项目地址，手机请注意下载arm版本，一般为arm64，不能直接在Linux系统中配置，必须得在Termux中操作</span></span><br><span class="line">[https://github.com/fatedier/frp/releases](https://github.com/fatedier/frp/releases)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机温度，需要除以1000</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/thermal/thermal_zone0/temp</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机电池容量，需root</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/power_supply/battery/capacity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置termux的启动操作</span></span><br><span class="line">vim ~/.bashrc</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">sshd <span class="comment"># 开启ssh服务，便于连接</span></span><br><span class="line"><span class="comment"># 配置frp相关操作，便于直接远程连接</span></span><br><span class="line"><span class="comment"># 配置进入Linux系统操作</span></span><br><span class="line">`</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Android</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Android</tag>
+        <tag>Termux</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>对各种电子产品的拓展玩法与互联</title>
+    <url>/hexo_blog/2019/09/14/IOT/%E5%AF%B9%E5%90%84%E7%A7%8D%E7%94%B5%E5%AD%90%E8%AE%BE%E5%A4%87%E7%9A%84%E6%8B%93%E5%B1%95%E7%8E%A9%E6%B3%95%E4%B8%8E%E4%BA%92%E8%81%94/</url>
+    <content><![CDATA[<p>介绍手中各种电子产品及各自的拓展玩法</p>
+<span id="more"></span>
+<h2
+id="对各种电子产品的拓展玩法与互联">对各种电子产品的拓展玩法与互联</h2>
+<p>手上有的电子产品，其中大部分应该是每个人都有的。</p>
 <ul>
-<li>越大的
-<code>block size</code>表示拆分后的矩阵个数变少，这样访问全局内存的次数会更多。但是，每个矩阵比较大，这样线程的并行度会更高。即，IO变小，计算变快。</li>
-<li>越小的
-<code>block size</code>表示拆分后的矩阵个数变多，这样访问全局内存的次数会更少。但是，每个矩阵比较小，这样线程的并行度会更低。即，IO变大，计算变慢。</li>
+<li><p>魅族手机</p></li>
+<li><p>笔记本电脑（Win10）</p></li>
+<li><p>1T的机械移动硬盘</p></li>
+<li><p>若干U盘。。</p></li>
+<li><p>Kindle Paper White 3</p></li>
+<li><p>树莓派Zero</p></li>
+<li><p>IPad 2018（IOS13前）</p></li>
 </ul>
-<h2 id="算子融合">算子融合</h2>
-<p>算子融合是将多个计算操作合并为一个计算操作，以减少计算量和内存访问次数，从而提高计算效率。比如，矩阵乘法和
-softmax 的融合，可以减少一次内存访问。</p>
-<h3 id="softmax-的改造">softmax 的改造</h3>
-<p>softmax 的计算公式为：</p>
-<p><span class="math display">\[
-\text{softmax}(x_i) = \frac{e^{x_i}}{\sum_{j=1}^n e^{x_j}}
-\]</span></p>
-<p>而通常为了数值稳定性（避免溢出），会先计算最大值，再减去最大值，最后再计算
-softmax。即：</p>
-<p><span class="math display">\[
-\text{softmax}(x_i) = \frac{e^{x_i - \max(x)}}{\sum_{j=1}^n e^{x_j -
-\max(x)}}
-\]</span></p>
-<p>在神经网络中，其值通常为一个矩阵（不只是一个维度），所以需要对每一行进行
-softmax。示例代码如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_softmax</span>(<span class="params">X, dim=<span class="number">1</span></span>):</span><br><span class="line">    X -= torch.<span class="built_in">max</span>(X, dim=dim, keepdim=<span class="literal">True</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> torch.exp(X) / torch.<span class="built_in">sum</span>(torch.exp(X), dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(my_softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), my_softmax(X, dim=<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
-<p>但这样跟矩阵乘法进行算子融合是没有优势的，两个函数还是独立计算的。算子融合是需要糅合两种计算操作。所以，需要把
-softmax
-函数放到矩阵乘法中进行计算。即将计算过程变成一个可迭代的过程，换而言之，随着元素的增加不断更新
-softmax 的结果。</p>
-<p><a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
-calculation for softmax</a>
-提供了这么一种方式。简单起见，从一个元素开始介绍。现在有一个列表，里面只有一个元素
-<code>[1]</code>，其 softmax 计算过程为：</p>
+<h3 id="手机与ipad">手机与IPad</h3>
+<p>这两个放在一起的原因是，在IOS13之前，IPad其实差不多可以等同于大尺寸IPone。所以这里就是一个安卓手机（未ROOT）与IPone的介绍，这里的玩法也算是比较多的，下面一一介绍。关于APP的安装包在此就不提供了，请读者自行搜索。当然还有ADB这种玩法，也不做介绍，可参考
+<a href="#">Post not found: IOT/用ADB调试安卓手机.md</a></p>
+<h4 id="termux与juicessh">Termux与JuiceSSH</h4>
+<p>放在搜索引擎一搜，其实这两个APP就有很多相关介绍了。但这里我想介绍一些不一样的玩法。关于Termux操作不方便这一点，可以使用JuiceSSH连接localhost来操作，具体操作不介绍。</p>
+<p><strong>下载</strong>：如何在手机上实现多线程下载？下一个ADM？当然这样很方便，但我们同样可以使用Termux来完成这一点。Termux安装Axel（<code>pkg install axel</code>），下载<code>axel -n 32 url</code>，使用32个线程下载链接。建议先<code>cd /sdcard/</code>中。</p>
+<p><strong>数据传输</strong>：一个手机和一个IPad，如何在不借助其他介质的前提下快速传输。QQ、微信、快牙等传输文件工具都行，但这里我自己实测速度都不如这里的方法快。就拿快牙举例，它实际是用手机开放一个<code>2.4G</code>的热点，然后用二维码进行链接传输。同样的，我们用同样的思路，手机开放<code>5G</code>热点，IPad手动连接此WIFI。打开Termux，先<code>ifconfig</code>知道本地IP，形如<code>192.168.x.1</code>这种，再<code>cd</code>到传输文件的目录，利用Python的静态资源服务器<code>python3 -m http.server</code>，这样，资源就对局域网内设备（连接到手机热点的设备）开放。接下来操作IPad，由于IOS13之前，<code>Safari</code>不支持输入链接下载，所以这里推荐<code>Alook</code>与<code>Documents</code>（后者免费），在<code>Documents</code>中的浏览器输入刚刚的手机<code>ip:8000</code>，这样就能看到手机资源所在文件目录的所有文件，点击即可下载。实测单个文件速度大概比<code>2.4G</code>快3倍（无需流量）。当然这里手机和IPad也可以同时连接同一个WIFI，然后使用同上方法进行传输（速度取决于路由器和网络带宽）。</p>
+<p><strong>远程手机</strong>：如何在手机不连接WIFI，不连接数据线的前提下，用其他设备访问手机（手机与其他设备的实际距离也可以很远），前提手机能上网。这一方法个人暂没有特别用途，但<code>Just for fun</code>。首先，需要一个公网IP服务器；接下来需要会玩ssh，<code>Termux</code>与服务器配置好ssh，<code>Termux</code>能够ssh到服务器上即可。最后一步，参考
+<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>。</p>
+<h4 id="spacedesk">spacedesk</h4>
+<p>电脑投屏，使用USB数据线连接设备和电脑，用这个软件进行投屏。我觉得挺鸡肋的。。。安卓手机屏幕又太小，测试IPad的时候发现闪退。</p>
+<h4
+id="androidhttpcapture和httpcanary">AndroidHttpCapture和HttpCanary</h4>
+<p>安卓抓包工具，后者的功能比前者更强大一些。有时候身边没有电脑，需要手机抓包看数据，所以就只能直接用手机看数据包了。前者基本满足了大部分场景的抓包数据。但当某些时候（比如某些APP的数据交互不是使用HTTP等常用协议，可能就没办法抓包，此时电脑上没办法直接抓包），所以就需要使用<code>HttpCanary</code>（需要ROOT）来对网络进行代理，在解析数据。测试如果电脑上使用<code>WireShark</code>，确实是可以抓到包的，但是会乱码。所以可能使用<code>HttpCanary</code>还更为方便一些。</p>
+<h4 id="netx与net-analyzer">NetX与Net Analyzer</h4>
+<p>安卓局域网设备分析工具，用的不多，具体不介绍，有兴趣自行下载、</p>
+<h4 id="ios同类软件免费">IOS同类软件，免费</h4>
+<ul>
+<li><p><code>iTerminal</code>，同<code>JuiceSSH</code></p></li>
+<li><p><code>Stream</code>，同<code>AndroidHttpCapture</code></p></li>
+<li><p><code>Fing</code>，同<code>NetX</code></p></li>
+</ul>
+<h4 id="scrcpy">scrcpy</h4>
+<p>在电脑上使用，安卓投屏工具，<a
+href="https://github.com/Genymobile/scrcpy">scrcpy</a>，由于魅族手机一些独特的操作，所以需要替换jar文件，<a
+href="https://github.com/taie02/taie02.github.io/tree/master/2019/03/03/scrcpy_usage">此项目</a>。</p>
+<h4 id="otg">OTG</h4>
+<p>关于OTG这一项，其实是属于外接设备这一项，通过充电接口转USB接口可以接大部分usb设备，比如u盘这种。甚至还可以反向充电，接鼠标键盘等。</p>
+<h3 id="u盘与移动硬盘">U盘与移动硬盘</h3>
+<p>二者都属于移动存储介质，其实也没什么好介绍的。。。可以提几点，Windows
+To
+Go（WTG）可以搞（高性能的移动盘），自己没尝试过。在移动硬盘上装了两个Linux系统。</p>
+<p>关于如何装系统就不介绍了，这里只有一点需要注意的是备份好数据再分区，在装系统的时候引导装载移动硬盘上，要不然移动硬盘上只能在装系统的设备上启动。</p>
+<h3 id="kindle">Kindle</h3>
+<p>这里不介绍越狱之类的操作，只有常规操作。。。</p>
+<ul>
+<li><p><a
+href="https://bookfere.com/post/19.html">搭建个人推送网站</a></p></li>
+<li><p><a href="https://volmoe.com/">漫画</a></p></li>
+<li><p>Chrome插件：Push to Kindle</p></li>
+</ul>
+<h3 id="树莓派">树莓派</h3>
+<p>吃灰派。。。大概也就能跑跑小demo，爬数据，利用空闲网络带宽下东西。同样也可以利用ssh，进行端口转发，远程连接。</p>
+<p>后续再更。。。</p>
+]]></content>
+      <categories>
+        <category>Life</category>
+      </categories>
+      <tags>
+        <tag>Raspberry</tag>
+        <tag>Linux</tag>
+        <tag>Android</tag>
+        <tag>Windows</tag>
+        <tag>IOS</tag>
+        <tag>Kindle</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>用ADB调试安卓手机</title>
+    <url>/hexo_blog/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</url>
+    <content><![CDATA[<p>使用adb工具实现对安卓手机的调试，包括安装应用、查看运行App、录屏、截图等功能。本文详细介绍了adb工具的安装、配置和使用方法，并列出了一些实用的命令，同时还介绍了无线网调试设备的方法。</p>
+<span id="more"></span>
+<p>写在开头，一些推荐阅读的博客文章</p>
+<p><a
+href="http://blog.csdn.net/fasfaf454/article/details/51438743?locationNum=14">通过adb获取手机信息</a></p>
+<p><a href="https://www.jianshu.com/p/5980c8c282ef">Android
+adb你真的会用吗?</a></p>
+<p><a
+href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
+<h3 id="介绍">介绍</h3>
+<p>adb是安卓开发调试工具，接触到这个还是因为微信跳一跳的外挂原因。看文档说明之后得知需要adb工具，进行简单尝试后，发现这玩意还真的是挺好用的。填补了自己用代码控制安卓手机的空缺（之前也玩过按键精灵，但是需要Root权限）。</p>
+<p><a
+href="https://developer.android.com/studio/command-line/adb.html?hl=zh-cn"><strong>adb官方说明</strong></a>：</p>
+<p>Android 调试桥 (adb)
+是一个通用命令行工具，其允许您与模拟器实例或连接的 Android
+设备进行通信。它可为各种设备操作提供便利，如安装和调试应用，并提供对
+Unix
+shell（可用来在模拟器或连接的设备上运行各种命令）的访问。该工具作为一个客户端-服务器程序，包括三个组件：</p>
+<h3 id="安装">安装</h3>
+<p>可参考文章: <a
+href="https://www.xda-developers.com/install-adb-windows-macos-linux/">How
+to Install ADB on Windows, macOS, and Linux</a></p>
+<h4 id="windows">Windows</h4>
+<p>这里我使用的是win10。如果以下操作不能成功配置，可以百度其他的安装教程</p>
 <ol type="1">
-<li>exp: 计算 <span class="math inline">\(e^{1}\)</span>，得到 <span
-class="math inline">\([e^{1}]\)</span></li>
-<li>max: 计算 <code>max(1)</code>，得到 <code>1</code></li>
-<li>-max: 计算 $e^{1 - 1} $，得到 <span class="math inline">\([e^{1 - 1}
-]\)</span></li>
-<li>sum: 计算 $e^{1- 1} $ 的和，得到 $e^{1- 1} $</li>
-<li>softmax: 计算 <span class="math inline">\([e^{1- 1} ] / (e^{1- 1}
-)\)</span>，得到 <span class="math inline">\([1]\)</span></li>
-</ol>
-<p>现在增加一个元素，观察有哪些变化。假设增加的元素为
-2，现在有一个列表，里面有两个元素 <code>[1, 2]</code>，新增的 softmax
-计算过程为：</p>
+<li><p>下载。<a
+href="https://dl.google.com/android/repository/platform-tools-latest-windows.zip">下载链接</a></p></li>
+<li><p>移动目录</p>
 <ol type="1">
-<li>exp: 计算 <span class="math inline">\(e^{2}\)</span>，得到 <span
-class="math inline">\(e^{2}\)</span></li>
-<li>max: 与原来的 <code>max(1)=1</code> 比较，得到
-<code>max(1, 2)</code>，得到 <code>2</code></li>
-<li>-max：所有元素更新一遍，均减去最新的 <code>max(1, 2)=2</code>，得到
-<span class="math inline">\([e^{1- 2} , e^{2- 2} ]\)</span></li>
-<li>sum: 重新计算一遍结果</li>
-<li>softmax: 计算 <span class="math inline">\([e^{1- 2}, e^{2 - 2}] /
-(e^{1 - 2} + e^{2 - 2})\)</span></li>
-</ol>
-<p>对比发现，第 3 步会导致重新第 4
-步计算一遍求和结果，但这个求和结果在第 5
-步中作为分母是可以灵活调整的。原来是<span
-class="math inline">\(e^{1-1}\)</span>，现在更新为<span
-class="math inline">\(e^{1-2}+e^{2-2}\)</span>。假设原来的最大值为
-<code>old_max</code>，新的最大值为 <code>new_max</code>，原来的元素为
-<code>old_v</code>，则原来元素的 <code>exp</code>结果可以更新为</p>
-<p><span class="math display">\[
-e^{old\_v-old\_max} \rightarrow e^{old\_v-old\_max+old\_max-new\_max} =
-e^{old\_v-old\_max} \times e^{old\_max-new\_max}
-\]</span></p>
-<p>而对于新加的元素，则可以在求和部分直接加上<span
-class="math inline">\(e^{2-2}\)</span>，所以，代码如下</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">nums = [<span class="number">1</span>, <span class="number">2</span>]</span><br><span class="line">sum_, max_v = <span class="number">0.</span>, <span class="number">0.</span></span><br><span class="line">norm_v = <span class="number">0.</span></span><br><span class="line"><span class="keyword">for</span> i, num <span class="keyword">in</span> <span class="built_in">enumerate</span>(nums):</span><br><span class="line">    old_max_v = max_v</span><br><span class="line">    max_v = <span class="built_in">max</span>(max_v, num)</span><br><span class="line">    norm_v = norm_v*np.exp(old_max_v - max_v) + np.exp(num - max_v)</span><br><span class="line">softmax_nums = [np.exp(num-max_v)/ norm_v <span class="keyword">for</span> num <span class="keyword">in</span> nums] </span><br></pre></td></tr></table></figure>
-<p>对于两个元素这个公式是适用的，那么对于多个元素呢？答案自然也是同样适用的</p>
-<p><span class="math display">\[
-\begin{aligned}
-e^{v1-old\_max}+e^{v2-old\_max} &amp;\rightarrow
-e^{v1-new\_max}+e^{v2-new\_max} \\ &amp;=
-e^{v1-old\_max+old\_max-new\_max}+e^{v2-old\_max+old\_max-new\_max}
-\\  \\ &amp;= e^{v1-old\_max} \times e^{old\_max-new\_max} +
-e^{v2-old\_max} \times e^{old\_max-new\_max} \\ &amp;= (e^{v1-old\_max}
-- e^{v2-old\_max}) \times e^{old\_max-new\_max}
-\end{aligned}
-\]</span></p>
-<p>所以，将其更新为二维矩阵的形式，就有</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">online_softmax</span>(<span class="params">X</span>):</span><br><span class="line">    value = torch.zeros_like(X)</span><br><span class="line">    <span class="keyword">for</span> row <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span></span><br><span class="line">        <span class="keyword">for</span> col <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">1</span>]):</span><br><span class="line">            val = X[row, col]</span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        value[row, :] = torch.exp(X[row, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> value</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(online_softmax(X))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), online_softmax(X))</span><br></pre></td></tr></table></figure>
-<h3 id="softmax-矩阵乘法">softmax + 矩阵乘法</h3>
-<p>一般的融合操作是需要在矩阵乘法后，计算结果的 softmax。即</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line">output = torch.softmax(A1 @ A2, dim=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>在结合上面的内容后，将其改造为</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(A1.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span>  </span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(A2.shape[<span class="number">1</span>]):</span><br><span class="line">            val = output[i, j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br><span class="line">          </span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        output[i, :] = torch.exp(output[i, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
-<p>更进一步，结合 Tiled matmul，将其改造为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]            </span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">tiled_matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">    block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        old_row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        normalizer_term = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 这里算完了每个block的结果，所以需要将其拆分成每个block，然后再计算softmax</span></span><br><span class="line">            <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):            </span><br><span class="line">                <span class="keyword">for</span> jj, col <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_j, end_j)):</span><br><span class="line">                    val = output[row][col]</span><br><span class="line">                    old_row_max[ii][jj] = row_max[ii][jj]</span><br><span class="line">                    row_max[ii][jj] = <span class="built_in">max</span>(old_row_max[ii][jj], val)</span><br><span class="line">                    normalizer_term[ii][jj] = normalizer_term[ii][jj] * np.exp(old_row_max[ii][jj] - row_max[ii][jj]) + np.exp(val - row_max[ii][jj])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):</span><br><span class="line">            row_max_v, _ = torch.<span class="built_in">max</span>(row_max, dim=<span class="number">1</span>)</span><br><span class="line">            <span class="comment"># 重算 sum, 代入公式 old_v*exp(old_max - new_max)</span></span><br><span class="line">            sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br><span class="line">            output[row, :] = torch.exp(output[row, :] - row_max_v[ii]) / sum_</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(tiled_matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), tiled_matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
-<p>由于这里是每次计算出来是一个 block ，所以要把 block
-拆分出每个元素，计算 block 中每行每列的最大值 row_max 以及分母
-normalizer_term。</p>
-<p>最后在计算 softmax 时，要计算所有 block 的最大值
-<code>torch.max(row_max, dim=1)</code>，还需要计算分母，这里需要考虑所有的
-block，可以类比于合并计算 <code>[1, 2]</code>,
-<code>[3, 4]</code>两个列表的normalizer_term 。已知对应的
-normalizer_term <span class="math inline">\(e^{1-2}+e^{2-2}\)</span> 和
-<span class="math inline">\(e^{3-4}+e^{3-4}\)</span>，合并后的结果应当是
-<span
-class="math inline">\(e^{1-4}+e^{2-4}+e^{3-4}+e^{3-4}\)</span>。将其公式化写作：</p>
-<p><span class="math display">\[
-\begin{aligned}
-\sum_{i=1}^{m} e^{x_i-max(x)} + \sum_{i=1}^{n} e^{y_i-max(y)} &amp;
-\rightarrow \sum_{i=1}^{m} e^{x_i-max(x,y)} + \sum_{i=1}^{n}
-e^{y_i-max(x,y)} \\ &amp;= \sum_{i=1}^{m}
-e^{x_i-max(x)}e^{max(x)-max(x,y)} + \sum_{i=1}^{n}
-e^{y_i-max(x,y)}e^{max(y)-max(x,y)}
-\end{aligned}
-\]</span></p>
-<p>假设 <code>max(x,y)=max(x)</code>，那么对应项将乘
-1，这并不会对结果有任何影响。对应的代码为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br></pre></td></tr></table></figure>
-<p>至此，完成了矩阵乘法和 softmax 的融合。接下来，会实现 cuda
-版本以对比性能。</p>
-<h3 id="cuda-实现">Cuda 实现</h3>
-<p>首先，接着之前矩阵乘法的cuda实现，</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">    <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">        value += input1[row][k] * input2[k][col];</span><br><span class="line">    &#125;</span><br><span class="line">    output[row][col] = value</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>在循环之后，已经计算完了输出矩阵中的一项。需要继续算每行的
-<code>row_max</code>和 <code>normalizer_term</code>。但由于这里是 cuda
-中的某个 block，所以需要借助共享内存来通信每行的结果。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">__shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">__shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">row_max[threadIdx.y][threadIdx.x] = value; <span class="comment">// 先把计算结果放到 row_max 中，以便于比较大小</span></span><br><span class="line">__syncthreads(); <span class="comment">// 这行代码是为了保证每个线程都已经计算完了，才能进行下一步的操作</span></span><br></pre></td></tr></table></figure>
-<p>计算过程分为三个步骤：1. 找到每行的最大值</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li>计算每行的 softmax 的分母每项组成</li>
-</ol>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line">normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line">__syncthreads();</span><br></pre></td></tr></table></figure>
-<ol start="3" type="1">
-<li>计算每行的 softmax 的每项之后</li>
+<li>解压压缩包</li>
+<li>移动文件夹你想让它放置的位置</li>
+</ol></li>
+<li><p>环境配置</p>
+<ol type="1">
+<li>添加环境变量，这样可以在cmd或者powershell中直接使用adb</li>
+<li>右键"计算机"-&gt;"属性"-&gt;左边的"高级系统设置"-&gt;"环境变量"&gt;-在上面用户变量中的"Path"进行编辑-&gt;"添加"刚刚放置的文件夹目录，确认保存即可
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135305187.png" /></li>
+<li>这样再打开cmd或者powershell，就可以直接使用adb命令了</li>
+</ol></li>
 </ol>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 最后将其更新到输出矩阵中</span></span><br><span class="line">output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br></pre></td></tr></table></figure>
-<p>完整代码如下（这里没有实现完整的attention，仅仅是一个矩阵乘法 +
-softmax 计算）：</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">        __shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        __shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = value;</span><br><span class="line">        __syncthreads();</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax 的分母</span></span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line"></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 计算每个 row  normalizer_term之和</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax</span></span><br><span class="line">        output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 这里偷懒没有换名字</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores&#125;;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>测试代码如下</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> mulsoftmax</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">seed = <span class="number">42</span></span><br><span class="line">torch.manual_seed(seed)</span><br><span class="line">torch.cuda.manual_seed(seed)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_py_softmax</span>(<span class="params">x, dim</span>):</span><br><span class="line">    e = torch.exp(x)</span><br><span class="line">    s = torch.<span class="built_in">sum</span>(e, dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">return</span> e / s</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_mulsoft</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="comment"># print(q@k.T)</span></span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_mulsoft(q, k, v)</span><br><span class="line">    cpp_values = mulsoftmax.forward(q, k, v)[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">loop=<span class="number">100</span></span>):</span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_mulsoft(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: mulsoftmax.forward(q, k, v)[<span class="number">0</span>], number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">16</span>, <span class="number">40</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="comment"># 先检查结果是否正确</span></span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    q, k, v = torch.rand(size=(m, n)), torch.rand(size=(m, n)), torch.rand(size=(m, n))</span><br><span class="line">    <span class="comment"># 循环1w次，对比性能差距</span></span><br><span class="line">    compare_time(<span class="number">10000</span>)</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
+<h4 id="linux">Linux</h4>
+<p>Linux系统安装adb工具还是很容易的，我使用的是Ubuntu17.10</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> apt updagte</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt upgrade</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt install android-tools-adb</span><br></pre></td></tr></table></figure>
+<h3 id="配置">配置</h3>
+<p>手机需要打开USB调试模式，在实验的时候，市面上流行的大部分安卓手机USB调式模式开关默认关闭。当然如果手机默认开启了USB调试开关，自然可以省去下面一步。这里简单介绍一下如何手动打开USB调试模式开关，手机“设置”—&gt;"关于手机"—&gt;点击"版本号"7次。每个手机开启的方式大同小异，如果有不同，可以百度“xx手机如何开启USB调试”。</p>
+<p>在USB调试开关开启之后，进入USB调试，打开USB调试开关，如下图。</p>
+<p><img src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135319543.png" width="300px" /></p>
+<p>之后使用USB数据线连接到电脑。</p>
+<p>连接成功后，手机需要信任电脑。电脑打开cmd/powershell（Windows）或者终端（Linux），输入
+<code>adb devices</code>。查看输出。</p>
+<p>如果有输出设备名且无警告，则表示连接成功，否则需要进行下面一步的配置。</p>
+<p><strong>Windows</strong></p>
+<p>首先需要连接手机，确保手机能连接到电脑且打开了USB调试。</p>
+<p>第二步，右键"计算机"-&gt;"属性"-&gt;左边的"设备管理器"-&gt;"便携设备"-&gt;"详细信息"-&gt;"属性"下拉选择"硬件Id"，查看"值"中的内容。</p>
+<p>第三步，进入C盘用户目录下的隐藏文件夹".android"，编辑或者新建文件"adb_usb.ini"，用记事本打开，在里面写上内容。我这里需要写的是"0x2A45"，0x是前缀不需要更改，后面的字符串对应的更改"硬件Id"里面"值"的VID后面的四个字符。保存退出即可</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135348748.png" /></p>
+<p>第四步，断开USB连接，重新连接，再输入
+<code>adb devices</code>。正常情况下应该会出现设备的成功连接的信息</p>
+<p><strong>Linux</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先连接手机，输入下面命令，找到关于连接手机的那一行</span></span><br><span class="line">&gt; lsusb</span><br><span class="line"><span class="comment"># 输出形如下面，最后的&quot;Google Inc&quot;表示的是设备名，这里需要找到匹配自己手机的设备，并记录好ID</span></span><br><span class="line">...</span><br><span class="line">Bus 001 Device 017: ID 18d1:d002 Google Inc.</span><br><span class="line">...</span><br><span class="line"><span class="comment"># 在目录/etc/udev/rules.d/下，添加文件70-android.rules(名字貌似无所谓？？？)</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 写入下面内容，idVendor表示ID冒号前的数字，idProduct表示的是ID冒号后的数字，MODE固定为&quot;0666&quot;，其他不变</span></span><br><span class="line">SUBSYSTEM==<span class="string">&quot;usb&quot;</span>, ATTRS&#123;idVendor&#125;==<span class="string">&quot;18d1&quot;</span>, ATTRS&#123;idProduct&#125;==<span class="string">&quot;d002&quot;</span>,MODE=<span class="string">&quot;0666&quot;</span></span><br><span class="line"><span class="comment"># 保存退出，修改权限</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chmod</span> 644   /etc/udev/rules.d/70-android.rules</span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chown</span> root. /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 重启服务</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service udev restart</span><br><span class="line">&gt; <span class="built_in">sudo</span> killall adb</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输入adb devices查看输出，是否有误</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="使用">使用</h3>
+<p>详细可参考这篇博客: <a
+href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
+<p>在这里贴出一些我认为比较有意思且实用的功能</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看所有连接设备</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行截图保存在sd卡的根目录下，名字为screen.png</span></span><br><span class="line">&gt; adb shell screencap -p /sdcard/screen.png</span><br><span class="line"><span class="comment"># 将截图发送到本地（当前目录下），也可以发送其他文件</span></span><br><span class="line">&gt; adb pull /sdcard/screen.png</span><br><span class="line"><span class="comment"># 删除本地文件</span></span><br><span class="line">&gt; adb shell <span class="built_in">rm</span> /sdcard/screen.png</span><br><span class="line"><span class="comment"># 发送电脑里的文件到设备</span></span><br><span class="line">&gt; adb shell push screen.png /sdcard/</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入手机的交互环境，操作类似linux终端，exit或者Ctrl+C退出</span></span><br><span class="line">&gt; adb shell</span><br><span class="line"></span><br><span class="line"><span class="comment"># 点击手机屏幕(1000,1000)的位置</span></span><br><span class="line">&gt; adb shell input tap 1000 1000</span><br><span class="line"><span class="comment"># 输入字符串&quot;helloworld&quot;，此处不能直接输入中文，且字符串不能有空格</span></span><br><span class="line">&gt; adb shell input text helloworld</span><br><span class="line"><span class="comment"># 滑动屏幕，从（100, 100)到（1000,1000)，经历10s(也可以当作长按屏幕来使用)</span></span><br><span class="line">&gt; adb shell input swipe 100 100 1000 1000 10</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看当前运行的App, 这里Windows没有grep所以会运行失败，可以进入先进入交互环境再输入下面去掉&quot;adb shell&quot;命令</span></span><br><span class="line">&gt; adb shell dumpsys window | grep mCurrentFocus</span><br><span class="line"><span class="comment"># 或者</span></span><br><span class="line">&gt; adb shell dumpsys activity activities | grep mFocusedActivity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 按下电源键</span></span><br><span class="line">&gt; adb shell input keyevent 26</span><br><span class="line"><span class="comment"># 按下返回键</span></span><br><span class="line">&gt; adb shell input keyevent 4</span><br><span class="line"><span class="comment"># 按下HOME健</span></span><br><span class="line">&gt; adb shell input keyevent 3</span><br><span class="line"><span class="comment"># 点亮屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 224</span><br><span class="line"><span class="comment"># 熄灭屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 223</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机安装了哪些App，输出按行输出App的包名</span></span><br><span class="line">&gt; adb shell pm list packages</span><br><span class="line"><span class="comment"># 加&quot;-s&quot;表示只输出系统应用</span></span><br><span class="line"><span class="comment"># 加&quot;-3&quot;表示只输出第三方应用</span></span><br><span class="line"><span class="comment"># 加字符串表示过滤应用名称，当然也可以使用grep</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装apk</span></span><br><span class="line">&gt; adb install &lt;packagename&gt;</span><br><span class="line"><span class="comment"># 卸载apk</span></span><br><span class="line">&gt; adb uninstall &lt;packagename&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 从桌面启动app</span></span><br><span class="line">&gt; adb shell monkey -p &lt;packagename&gt; -c android.intent.category.LAUNCHER 1</span><br><span class="line"><span class="comment"># 关闭app</span></span><br><span class="line">&gt; adb shell am force-stop &lt;packagename&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="使用无线网调试设备">使用无线网调试设备</h3>
+<p>确保手机和电脑在同一局域网内</p>
+<p>如何查看手机的ip，由于每个设备的方法不大相同，在此不进行说明，请读者自行百度。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用USB连接设备，启动手机5555端口</span></span><br><span class="line">&gt; adb tcpip 5555</span><br><span class="line"><span class="comment"># adb连接命令，需要知道手机的ip</span></span><br><span class="line">&gt; adb connect 192.168.1.111</span><br><span class="line"><span class="comment"># 输入如下则表示连接成功</span></span><br><span class="line">connected to 192.168.1.111:5555</span><br><span class="line"><span class="comment"># 这里要注意的是，如果接下来的步骤未断掉数据线，则电脑是连接了两台设备的，进行操作会产生错误，所以可以拔掉数据线或者断开网络或者指定设备运行</span></span><br><span class="line"><span class="comment"># 断开无线网连接</span></span><br><span class="line">&gt; adb disconnect 192.168.1.111</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Android</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>ADB</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Matplotlib笔记</title>
+    <url>/hexo_blog/2017/08/26/PythonPackages/Matplotlib%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>解决Python中文乱码问题的完整方案及Matplotlib绘图基础操作</p>
+<span id="more"></span>
+<h1 id="中文乱码解决方案">中文乱码解决方案</h1>
+<p>支持所有平台的解决方案</p>
+<h2 id="定位默认字体位置">定位默认字体位置</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先进入python的交互环境</span></span><br><span class="line">&gt;&gt;&gt;python</span><br><span class="line"></span><br><span class="line">&gt;from matplotlib.font_manager import findfont, FontProperties</span><br><span class="line"><span class="comment"># 当前使用的默认字体</span></span><br><span class="line">&gt;findfont(FontProperties(family=FontProperties().get_family()))</span><br><span class="line"><span class="comment"># 在此记住返回值</span></span><br></pre></td></tr></table></figure>
+<h2 id="下载支持中文的字体">下载支持中文的字体</h2>
+<p>在这里推荐微软雅黑，msyh.ttf。</p>
+<p><a
+href="http://www.monmonkey.com/sonota/font1/getfont.html">推荐链接</a></p>
+<h2 id="偷梁换柱">偷梁换柱</h2>
+<p>在刚刚查询的那个文件夹中，将原来的默认字体重命名，之后将我们下载msyh.ttf替换进去，并重命名为那个默认字体之前的名字</p>
+<h2 id="大功告成">大功告成</h2>
+<p>当然为了保险起见，还可以操作下面一步。到现在为止，可以测试看看是否成功输出中文。</p>
+<p>找到matplotlib的安装目录，一般是在python的安装目录下的-packages-data,之后修改matplotlibrc文件</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将下面这两行冒号后面的字符替换</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#font.sans-serif     : nothing</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#verbose.level  : debug</span></span><br></pre></td></tr></table></figure>
+<h1 id="matplotlib.pyplot-基础操作">matplotlib.pyplot 基础操作</h1>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> matplotlib.pyplot <span class="keyword">as</span> plt</span><br><span class="line"></span><br><span class="line">x_list = []</span><br><span class="line">y_list = []</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定画布</span></span><br><span class="line">plt.figure(figsize=(<span class="number">8</span>, <span class="number">6</span>), dpi=<span class="number">80</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画y_list的折线图，格式为&#x27;ro-&#x27;，颜色为blue</span></span><br><span class="line">plt.plot(y_list, <span class="string">&#x27;ro-&#x27;</span>, color=<span class="string">&#x27;blue&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴画折线图，前提x_list内容全为整型数字</span></span><br><span class="line">plt.plot(x_list, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴，x_list为字符串</span></span><br><span class="line">x = <span class="built_in">range</span>(<span class="built_in">len</span>(x_list))</span><br><span class="line">plt.plot(x, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定横坐标, 跟着上面的字符串x_list的设定，每个字符串旋转角度为45，字体大小为20</span></span><br><span class="line">plt.xticks(x, x_list, rotation=<span class="number">45</span>, fontsize=<span class="number">20</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定标题，注如果需要设定中文需要更改matplotlib的字体配置</span></span><br><span class="line">plt.title(<span class="string">&#x27;title&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 保存图片名为plt</span></span><br><span class="line">plt.savefig(<span class="string">&quot;plt.png&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 运行完打开图片</span></span><br><span class="line">plt.show()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 注：如果需要保存图片，切记需要在打开图片前执行命令，否则保存图片为空白</span></span><br></pre></td></tr></table></figure>
+<h1 id="搬运工系列">搬运工系列</h1>
+<p><a
+href="https://juejin.im/post/5a9e14726fb9a028b86d87c9">5种快速易用的Python
+Matplotlib数据可视化方法</a></p>
+<p><a
+href="http://www.labri.fr/perso/nrougier/teaching/matplotlib/">Matplotlib
+教程</a></p>
+<p><a
+href="https://liam0205.me/2014/09/11/matplotlib-tutorial-zh-cn/">Matplotlib
+教程（中文翻译版本）</a></p>
+]]></content>
+      <categories>
+        <category>Visualization</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>Matplotlib</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Numpy的70个操作</title>
+    <url>/hexo_blog/2018/03/02/PythonPackages/Numpy%E7%9A%8470%E4%B8%AA%E6%93%8D%E4%BD%9C/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Numpy从入门到进阶的操作,
+翻译文章，便于自己使用时查阅</p>
+<span id="more"></span>
+<p><a
+href="https://www.machinelearningplus.com/101-numpy-exercises-python/">原文链接</a></p>
+<p>注释及输出前带有 <code>#</code></p>
+<h2 id="section">1-10</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看numpy的版本</span></span><br><span class="line">np.__version__</span><br><span class="line"><span class="comment"># 1.13.3</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个一维数据</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个布尔型数组</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.full((<span class="number">3</span>, <span class="number">3</span>), <span class="literal">True</span>, dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.ones((<span class="number">3</span>, <span class="number">3</span>), dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># array([[ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True]], dtype=bool)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数组中为奇数的数字</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([1, 3, 5, 7, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中为奇数的数字为-1</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>] = -<span class="number">1</span></span><br><span class="line"><span class="comment"># 方法二, 更加安全的操作方法（这里算新的一个操作）：</span></span><br><span class="line">out = np.where(arr % <span class="number">2</span> == <span class="number">1</span>, -<span class="number">1</span>, arr)</span><br><span class="line">out</span><br><span class="line"><span class="comment"># array([ 0, -1,  2, -1,  4, -1,  6, -1,  8, -1])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组维度变换</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line">arr.reshape(<span class="number">2</span>, -<span class="number">1</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 纵向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">0</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.vstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.r_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 横向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.hstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.c_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4, 1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9, 1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据原有的数组，自由拓展</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]) <span class="comment"># 原来的数组</span></span><br><span class="line">np.r_[np.repeat(a, <span class="number">3</span>), np.tile(a, <span class="number">3</span>)]</span><br><span class="line"><span class="comment"># array([1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-1">11-20</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.intersect1d(a, b)</span><br><span class="line"><span class="comment"># array([2, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在a数组中移去a、b数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>])	<span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>])	<span class="comment"># 原来的数组b</span></span><br><span class="line">np.setdiff1d(a, b)</span><br><span class="line"><span class="comment"># array([1, 2, 3, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共元素（索引相同）的索引</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.where(a == b)</span><br><span class="line"><span class="comment"># (array([1, 3, 5, 7], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取指定数值区间的元素</span></span><br><span class="line">a = np.arange(<span class="number">15</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">index = np.where((a &gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">index = np.where(np.logical_and(a&gt;=<span class="number">5</span>, a&lt;=<span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法三</span></span><br><span class="line">a[(a&gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>)]</span><br><span class="line"><span class="comment"># array([ 5,  6,  7,  8,  9, 10])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy运用函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">maxx</span>(<span class="params">x, y</span>):		<span class="comment"># 需要使用的函数</span></span><br><span class="line">    <span class="keyword">if</span> x &gt;= y:</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="keyword">return</span> y</span><br><span class="line"></span><br><span class="line">a = np.array([<span class="number">5</span>, <span class="number">7</span>, <span class="number">9</span>, <span class="number">8</span>, <span class="number">6</span>, <span class="number">4</span>, <span class="number">5</span>]) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">6</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">7</span>, <span class="number">1</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">pair_max = np.vectorize(maxx, dtypes=[<span class="built_in">float</span>])</span><br><span class="line">pair_max(a, b)</span><br><span class="line"><span class="comment"># array([ 6.,  7.,  9.,  8.,  9.,  7.,  5.])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两行</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[:, [<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>]]</span><br><span class="line"><span class="comment"># array([[1, 0, 2],</span></span><br><span class="line"><span class="comment">#        [4, 3, 5],</span></span><br><span class="line"><span class="comment">#        [7, 6, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两列</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[[<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>], :]</span><br><span class="line"><span class="comment"># array([[3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2],</span></span><br><span class="line"><span class="comment">#        [6, 7, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每行逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[6, 7, 8],</span></span><br><span class="line"><span class="comment">#        [3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每列逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[2, 1, 0],</span></span><br><span class="line"><span class="comment">#        [5, 4, 3],</span></span><br><span class="line"><span class="comment">#        [8, 7, 6]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个二维数组，数值区间为5-10</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">rand_arr = np.random.randint(low=<span class="number">5</span>, high=<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>)) + np.random.random((<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">rand_arr = np.random.uniform(<span class="number">5</span>,<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[ 6.75709583,  7.4662972 ,  7.2395679 ],</span></span><br><span class="line"><span class="comment">#        [ 9.7802112 ,  6.70830431,  6.76776651],</span></span><br><span class="line"><span class="comment">#        [ 6.35564151,  7.8632512 ,  8.53040631],</span></span><br><span class="line"><span class="comment">#        [ 7.99457763,  7.85526309,  8.96584465],</span></span><br><span class="line"><span class="comment">#        [ 6.18621991,  8.14214732,  7.31162037]])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-2">21-30</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出数组时，控制输出精度</span></span><br><span class="line">rand_arr = np.random.random([<span class="number">3</span>,<span class="number">3</span>]) / <span class="number">1e3</span>  <span class="comment"># 原始数组</span></span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.43404942e-04,   2.78369385e-04,   4.24517591e-04],</span></span><br><span class="line"><span class="comment">#        [  8.44776132e-04,   4.71885619e-06,   1.21569121e-04],</span></span><br><span class="line"><span class="comment">#        [  6.70749085e-04,   8.25852755e-04,   1.36706590e-04]])</span></span><br><span class="line"><span class="comment"># 第一个操作，指定小数点后3位数</span></span><br><span class="line">np.set_printoptions(precision=<span class="number">3</span>)</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.434e-04,   2.784e-04,   4.245e-04],</span></span><br><span class="line"><span class="comment">#        [  8.448e-04,   4.719e-06,   1.216e-04],</span></span><br><span class="line"><span class="comment">#        [  6.707e-04,   8.259e-04,   1.367e-04]])</span></span><br><span class="line"><span class="comment"># 第二个操作，避免输出e-04，并指定小数点后6位数</span></span><br><span class="line">np.set_printoptions(suppress=<span class="literal">True</span>, precision=<span class="number">6</span>)</span><br><span class="line"><span class="comment"># array([[ 0.000543,  0.000278,  0.000425],</span></span><br><span class="line"><span class="comment">#        [ 0.000845,  0.000005,  0.000122],</span></span><br><span class="line"><span class="comment">#        [ 0.000671,  0.000826,  0.000137]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 含有很多元素的数组</span></span><br><span class="line">a = np.arange(<span class="number">15</span>)	<span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定显示元素个数的阈值</span></span><br><span class="line">np.set_printoptions(threshold=<span class="number">6</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2, ..., 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定不隐藏元素输出</span></span><br><span class="line">np.set_printoptions(threshold=np.nan)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 导入网站数据</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">iris[:<span class="number">3</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取某一列</span></span><br><span class="line">species = np.array([row[<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris])</span><br><span class="line">species</span><br><span class="line"><span class="comment"># array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;|S15&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取多列</span></span><br><span class="line"><span class="comment"># 方法一，在导入的时候指定列</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])</span><br><span class="line"><span class="comment"># 方法二，使用上面的方法</span></span><br><span class="line">iris = np.array([row.tolist()[:<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris], dtype=<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line"><span class="comment"># array([[ 5.1,  3.5,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  3. ,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.3,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.6,  3.1,  1.5,  0.2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算平均值、方差、中位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">mu, med, sd = np.mean(sepallength), np.median(sepallength), np.std(sepallength)</span><br><span class="line"><span class="comment"># 5.84333333333 5.8 0.825301291785</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 0-1化数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">Smax, Smin = sepallength.<span class="built_in">max</span>(), sepallength.<span class="built_in">min</span>()</span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">S = (sepallength - Smin)/(Smax - Smin)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">S = (sepallength - Smin)/sepallength.ptp()</span><br><span class="line"><span class="comment"># array([ 0.222222,  0.166667,  0.111111, ...,  0.611111,  0.527778,</span></span><br><span class="line"><span class="comment">#         0.444444])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># softmax数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">softmax</span>(<span class="params">x</span>):</span><br><span class="line">    e_x = np.exp(x - np.<span class="built_in">max</span>(x))</span><br><span class="line">    <span class="keyword">return</span> e_x / e_x.<span class="built_in">sum</span>(axis=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">softmax(sepallength)</span><br><span class="line"><span class="comment"># array([ 0.00222 ,  0.001817,  0.001488, ...,  0.009001,  0.006668,  0.00494 ])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-3">31-40</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数据中百分位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="comment"># 提取数据中5%, 95%的数</span></span><br><span class="line">np.percentile(sepallength, q=[<span class="number">5</span>, <span class="number">95</span>])</span><br><span class="line"><span class="comment"># array([ 4.6  ,  7.255])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数据中的缺失值替换为随机数</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">i, j = np.where(iris_2d)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.choice((i), <span class="number">20</span>), np.random.choice((j), <span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line">iris_2d[:<span class="number">10</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.6&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.4&#x27;, b&#x27;3.9&#x27;, b&#x27;1.7&#x27;, b&#x27;0.4&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.4&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, nan, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数据中缺失值位置</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 统计缺失值个数</span></span><br><span class="line">np.isnan(iris_2d[:, <span class="number">0</span>]).<span class="built_in">sum</span>()</span><br><span class="line"><span class="comment"># 5</span></span><br><span class="line"><span class="comment"># 统计缺失值位置</span></span><br><span class="line">np.where(np.isnan(iris_2d[:, <span class="number">0</span>]))</span><br><span class="line"><span class="comment"># (array([ 38,  80, 106, 113, 121], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组多重条件筛选</span></span><br><span class="line">condition = (iris_2d[:, <span class="number">2</span>] &gt; <span class="number">1.5</span>) &amp; (iris_2d[:, <span class="number">0</span>] &lt; <span class="number">5.0</span>)</span><br><span class="line">iris_2d[condition]</span><br><span class="line"><span class="comment"># array([[ 4.8,  3.4,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.4,  1.9,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.1,  1.6,  nan],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  2.4,  3.3,  1. ]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除有缺失值的行，注意~</span></span><br><span class="line">any_nan_in_row = np.array([~np.<span class="built_in">any</span>(np.isnan(row)) <span class="keyword">for</span> row <span class="keyword">in</span> iris_2d])</span><br><span class="line"><span class="comment"># 未删除前</span></span><br><span class="line">iris_2d.shape</span><br><span class="line"><span class="comment"># (150, 4)</span></span><br><span class="line"><span class="comment"># 删除后</span></span><br><span class="line">iris_2d[any_nan_in_row].shape()</span><br><span class="line"><span class="comment"># (130, 4)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算数组间关联度</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">np.corrcoef(iris[:, <span class="number">0</span>], iris[:, <span class="number">2</span>])[<span class="number">0</span>, <span class="number">1</span>]</span><br><span class="line"><span class="comment"># 0.87175415730487182</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 检查数组中是否含有缺失值</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中的缺失值为0</span></span><br><span class="line">iris_2d[np.isnan(iris_2d)] = <span class="number">0</span></span><br><span class="line"><span class="comment"># 替换前</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"><span class="comment"># 替换后</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># False</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数组中非重复数值的情况</span></span><br><span class="line">np.unique(iris, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># 上面的数组的表示数字，下面的数组表示出现次数</span></span><br><span class="line"><span class="comment"># (array([ 0.1,  0.2,  0.3,  0.4,  0.5,  0.6,  1. ,  1.1,  1.2,  1.3,  1.4,</span></span><br><span class="line"><span class="comment">#          1.5,  1.6,  1.7,  1.8,  1.9,  2. ,  2.1,  2.2,  2.3,  2.4,  2.5,</span></span><br><span class="line"><span class="comment">#          2.6,  2.7,  2.8,  2.9,  3. ,  3.1,  3.2,  3.3,  3.4,  3.5,  3.6,</span></span><br><span class="line"><span class="comment">#          3.7,  3.8,  3.9,  4. ,  4.1,  4.2,  4.3,  4.4,  4.5,  4.6,  4.7,</span></span><br><span class="line"><span class="comment">#          4.8,  4.9,  5. ,  5.1,  5.2,  5.3,  5.4,  5.5,  5.6,  5.7,  5.8,</span></span><br><span class="line"><span class="comment">#          5.9,  6. ,  6.1,  6.2,  6.3,  6.4,  6.5,  6.6,  6.7,  6.8,  6.9,</span></span><br><span class="line"><span class="comment">#          7. ,  7.1,  7.2,  7.3,  7.4,  7.6,  7.7,  7.9]),</span></span><br><span class="line"><span class="comment">#  array([ 6, 28,  7,  7,  1,  1,  8,  4,  7, 20, 20, 26, 11,  6, 12,  7,  7,</span></span><br><span class="line"><span class="comment">#          6,  6, 12,  6, 11,  5,  9, 14, 10, 27, 12, 13,  8, 12,  8,  4,  4,</span></span><br><span class="line"><span class="comment">#          7,  5,  6,  4,  5,  3,  8,  9,  7,  7,  9, 11, 14, 17,  6,  3,  8,</span></span><br><span class="line"><span class="comment">#         10, 12, 11, 10,  5,  8,  9,  4, 10,  8,  5,  3, 10,  3,  5,  1,  1,</span></span><br><span class="line"><span class="comment">#          3,  1,  1,  1,  4,  1], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数组中处于相应区间的数换位文本</span></span><br><span class="line"><span class="comment"># 将数组中数组位于[0, 3, 5, 10]区间内的转换为对应的索引</span></span><br><span class="line">petal_length_bin = np.digitize(iris[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>), [<span class="number">0</span>, <span class="number">3</span>, <span class="number">5</span>, <span class="number">10</span>])</span><br><span class="line">label_map = &#123;<span class="number">1</span>: <span class="string">&#x27;small&#x27;</span>, <span class="number">2</span>: <span class="string">&#x27;medium&#x27;</span>, <span class="number">3</span>: <span class="string">&#x27;large&#x27;</span>, <span class="number">4</span>: np.nan&#125;</span><br><span class="line">petal_length_cat = [label_map[x] <span class="keyword">for</span> x <span class="keyword">in</span> petal_length_bin]</span><br><span class="line">petal_length_cat[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># [&#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;]</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-4">41-50</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在已经存在的数组中，多加一列</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 计算</span></span><br><span class="line">sepallength = iris_2d[:, <span class="number">0</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">petallength = iris_2d[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">volume = (np.pi * petallength * (sepallength**<span class="number">2</span>))/<span class="number">3</span></span><br><span class="line">volume = volume[:, np.newaxis]</span><br><span class="line"><span class="comment"># 横向堆叠</span></span><br><span class="line">out = np.hstack([iris_2d, volume])</span><br><span class="line">out[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 38.13265162927291],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 35.200498485922445],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 30.0723720777127],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 33.238050274980004]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对数组进行概率抽样</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">species = iris[:, <span class="number">4</span>]</span><br><span class="line"><span class="comment"># 方法一：生成概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.array([<span class="string">&#x27;Iris-setosa&#x27;</span>, <span class="string">&#x27;Iris-versicolor&#x27;</span>, <span class="string">&#x27;Iris-virginica&#x27;</span>])</span><br><span class="line">species_out = np.random.choice(a, <span class="number">150</span>, p=[<span class="number">0.5</span>, <span class="number">0.25</span>, <span class="number">0.25</span>])</span><br><span class="line"><span class="comment"># 方法二：直接概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">probs = np.r_[np.linspace(<span class="number">0</span>, <span class="number">0.500</span>, num=<span class="number">50</span>), np.linspace(<span class="number">0.501</span>, <span class="number">.750</span>, num=<span class="number">50</span>), np.linspace(<span class="number">.751</span>, <span class="number">1.0</span>, num=<span class="number">50</span>)]</span><br><span class="line">index = np.searchsorted(probs, np.random.random(<span class="number">150</span>))</span><br><span class="line">species_out = species[index]</span><br><span class="line">np.unique(species_out, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># (array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-versicolor&#x27;, b&#x27;Iris-virginica&#x27;], dtype=object), array([77, 37, 36], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对筛选后的数组，取出第二大的数</span></span><br><span class="line">petal_len_setosa = iris[iris[:, <span class="number">4</span>] == <span class="string">b&#x27;Iris-setosa&#x27;</span>, [<span class="number">2</span>]].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">np.unique(np.sort(petal_len_setosa))[-<span class="number">2</span>]</span><br><span class="line"><span class="comment"># 1.7</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对二维数组按照其中某一列进行排序</span></span><br><span class="line"><span class="comment"># 按照第一列升序</span></span><br><span class="line">iris[iris[:,<span class="number">0</span>].argsort()][:<span class="number">20</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;4.3&#x27;, b&#x27;3.0&#x27;, b&#x27;1.1&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.0&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;2.9&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.5&#x27;, b&#x27;2.3&#x27;, b&#x27;1.3&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.6&#x27;, b&#x27;1.0&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.2&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.9&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.1&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.4&#x27;, b&#x27;3.3&#x27;, b&#x27;1.0&#x27;, b&#x27;Iris-versicolor&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.5&#x27;, b&#x27;4.5&#x27;, b&#x27;1.7&#x27;, b&#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计出现次数最多的数</span></span><br><span class="line">vals, counts = np.unique(iris[:, <span class="number">3</span>], return_counts=<span class="literal">True</span>)</span><br><span class="line">vals[np.argmax(counts)]</span><br><span class="line"><span class="comment"># b&#x27;0.2&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中第一次出现比给定数字大的值的缩影</span></span><br><span class="line"><span class="comment"># 这里设定要找比1.0大的值</span></span><br><span class="line">np.argmax(iris[:, <span class="number">3</span>].astype(<span class="string">&#x27;float&#x27;</span>) &gt; <span class="number">1.0</span>)</span><br><span class="line"><span class="comment"># 50</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一个区间，如果数字不在这个区间内，就替换为区间的最小/最大值</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.clip(a, a_min=<span class="number">10</span>, a_max=<span class="number">30</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.where(a &lt; <span class="number">10</span>, <span class="number">10</span>, np.where(a &gt; <span class="number">30</span>, <span class="number">30</span>, a))</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  30.        ,  30.        ,  17.46948556,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.26877027,  10.        ,  13.36889132,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.7152288 ,  10.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  29.26817347])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计前n大的元素位置</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">a.argsort()[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># array([ 2, 15, 18,  1, 16], dtype=int64)</span></span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.argpartition(-a, <span class="number">5</span>)[:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([16,  1, 18, 15,  2], dtype=int64)</span></span><br><span class="line"><span class="comment"># 获取前n大的元素</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法1:</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法2:</span></span><br><span class="line">np.sort(a)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法3:</span></span><br><span class="line">np.partition(a, kth=-<span class="number">5</span>)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 前三个输出都是</span></span><br><span class="line"><span class="comment"># array([ 41.06481956,  44.63016661,  47.06146116,  44.63675129,  49.06512199])</span></span><br><span class="line"><span class="comment"># 方法4:</span></span><br><span class="line">a[np.argpartition(-a, <span class="number">5</span>)][:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([ 49.06512199,  47.06146116,  44.63675129,  44.63016661,  41.06481956])</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算有唯一值的行数,没看懂</span></span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">11</span>,size=(<span class="number">6</span>, <span class="number">10</span>))</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[ 2,  6,  5,  3,  9,  4,  6,  1, 10,  4],</span></span><br><span class="line"><span class="comment">#        [ 7,  4,  5,  8,  7,  4, 10,  1,  5,  5],</span></span><br><span class="line"><span class="comment">#        [ 6,  8,  7,  7,  3,  5,  3,  8,  2,  7],</span></span><br><span class="line"><span class="comment">#        [ 7,  1,  8,  3,  4,  6,  5,  3,  5,  4],</span></span><br><span class="line"><span class="comment">#        [ 8, 10,  1,  1,  6, 10,  7,  7,  6,  7],</span></span><br><span class="line"><span class="comment">#        [ 5,  8,  4, 10,  3,  4,  9,  8,  2,  6]])</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">counts_of_all_values_rowwise</span>(<span class="params">arr2d</span>):</span><br><span class="line">    num_counts_array = [np.unique(row, return_counts=<span class="literal">True</span>) <span class="keyword">for</span> row <span class="keyword">in</span> arr2d]</span><br><span class="line">    <span class="keyword">return</span> [[<span class="built_in">int</span>(b[a==i]) <span class="keyword">if</span> i <span class="keyword">in</span> a <span class="keyword">else</span> <span class="number">0</span> <span class="keyword">for</span> i <span class="keyword">in</span> np.unique(arr2d)] <span class="keyword">for</span> a, b <span class="keyword">in</span> num_counts_array]</span><br><span class="line">counts_of_all_values_rowwise(arr)</span><br><span class="line"><span class="comment"># [[1, 1, 1, 2, 1, 2, 0, 0, 1, 1],</span></span><br><span class="line"><span class="comment">#  [1, 0, 0, 2, 3, 0, 2, 1, 0, 1],</span></span><br><span class="line"><span class="comment">#  [0, 1, 2, 0, 1, 1, 3, 2, 0, 0],</span></span><br><span class="line"><span class="comment">#  [1, 0, 2, 2, 2, 1, 1, 1, 0, 0],</span></span><br><span class="line"><span class="comment">#  [2, 0, 0, 0, 0, 2, 3, 1, 0, 2],</span></span><br><span class="line"><span class="comment">#  [0, 1, 1, 2, 1, 1, 0, 2, 1, 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将多维数组转换为一维数组</span></span><br><span class="line">arr1 = np.arange(<span class="number">3</span>)</span><br><span class="line">arr2 = np.arange(<span class="number">3</span>,<span class="number">7</span>)</span><br><span class="line">arr3 = np.arange(<span class="number">7</span>,<span class="number">10</span>)</span><br><span class="line">array_of_arrays = np.array([arr1, arr2, arr3])</span><br><span class="line">array_of_arrays      <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([array([0, 1, 2]), array([3, 4, 5, 6]), array([7, 8, 9])], dtype=object)</span></span><br><span class="line">arr_2d = np.concatenate(array_of_arrays)</span><br><span class="line">arr_2d</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-5">51-60</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 唯一编码位置</span></span><br><span class="line">np.random.seed(<span class="number">101</span>)</span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">4</span>, size=<span class="number">6</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([2, 3, 2, 2, 2, 1])</span></span><br><span class="line"><span class="comment"># 返回数组是3x6,因为数组长度为6，最大数为1。1对应的是[1,0,0,],2对应的是[0,1,0],3对应的是[0,0,1]</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">one_hot_encodings</span>(<span class="params">arr</span>):</span><br><span class="line">    uniqs = np.unique(arr)</span><br><span class="line">    out = np.zeros((arr.shape[<span class="number">0</span>], uniqs.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">for</span> i, k <span class="keyword">in</span> <span class="built_in">enumerate</span>(arr):</span><br><span class="line">        out[i, k-<span class="number">1</span>] = <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> out</span><br><span class="line">one_hot_encodings(arr)</span><br><span class="line"><span class="comment"># array([[ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  0.,  1.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 1.,  0.,  0.]])</span></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">(arr[:, <span class="literal">None</span>] == np.unique(arr)).view(np.int8)</span><br><span class="line"><span class="comment"># array([[0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 0, 1],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [1, 0, 0]], dtype=int8)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取由变量名分组的行号</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">species = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;str&#x27;</span>, usecols=<span class="number">4</span>)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"></span><br><span class="line">[i <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> i, grp <span class="keyword">in</span> <span class="built_in">enumerate</span>(species_small[species_small==val])]</span><br><span class="line">[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据已知分类组名创建索引</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"><span class="comment"># 方法一，用列表解析式</span></span><br><span class="line">output = [np.argwhere(np.unique(species_small) == s).tolist()[<span class="number">0</span>][<span class="number">0</span>] <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]]</span><br><span class="line"><span class="comment"># 方法二，不用列表解析式直接循环</span></span><br><span class="line">output = []</span><br><span class="line">uniqs = np.unique(species_small)</span><br><span class="line"><span class="keyword">for</span> val <span class="keyword">in</span> uniqs:  <span class="comment"># uniq values in group</span></span><br><span class="line">    <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]:  <span class="comment"># each element in group</span></span><br><span class="line">        groupid = np.argwhere(uniqs == s).tolist()[<span class="number">0</span>][<span class="number">0</span>]  <span class="comment"># groupid</span></span><br><span class="line">        output.append(groupid)</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数值排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=<span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [ 9  4 15  0 17 16 17  8  9  0]</span></span><br><span class="line">a.argsort().argsort()</span><br><span class="line"><span class="comment"># [4 2 6 0 8 7 9 3 5 1]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 多维数组排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=[<span class="number">2</span>,<span class="number">5</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line">a.ravel().argsort().argsort().reshape(a.shape)</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line"><span class="comment"># [[4 2 6 0 8]</span></span><br><span class="line"><span class="comment">#  [7 9 3 5 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一行的最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">np.amax(a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.apply_along_axis(np.<span class="built_in">max</span>, arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([9, 8, 6, 3, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每行最小值比最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line">np.apply_along_axis(<span class="keyword">lambda</span> x: np.<span class="built_in">min</span>(x)/np.<span class="built_in">max</span>(x), arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([ 0.44444444,  0.125     ,  0.5       ,  1.        ,  0.11111111])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中重复的数值，输出布尔型</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">0</span>, <span class="number">5</span>, <span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [0 0 3 0 2 4 2 2 2 2]</span></span><br><span class="line">out = np.full(a.shape[<span class="number">0</span>], <span class="literal">True</span>)</span><br><span class="line">unique_positions = np.unique(a, return_index=<span class="literal">True</span>)[<span class="number">1</span>]</span><br><span class="line">out[unique_positions] = <span class="literal">False</span></span><br><span class="line">out</span><br><span class="line"><span class="comment"># [False  True False  True False False  True  True  True  True]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一分类的平均值</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">numeric_column = iris[:, <span class="number">1</span>].astype(<span class="string">&#x27;float&#x27;</span>)  <span class="comment"># sepalwidth</span></span><br><span class="line">grouping_column = iris[:, <span class="number">4</span>]  <span class="comment"># species</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">output = [[group_val, numeric_column[grouping_column==group_val].mean()] <span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column)]</span><br><span class="line"><span class="comment"># 方法二：不用列表解析直接循环</span></span><br><span class="line">output = []</span><br><span class="line"><span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column):</span><br><span class="line">    output.append([group_val, numeric_column[grouping_column==group_val].mean()])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [[b&#x27;Iris-setosa&#x27;, 3.418],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-versicolor&#x27;, 2.770],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-virginica&#x27;, 2.974]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将图像转换为数组存储，并再转换出图像显示</span></span><br><span class="line"><span class="keyword">from</span> io <span class="keyword">import</span> BytesIO</span><br><span class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="keyword">import</span> PIL, requests</span><br><span class="line"></span><br><span class="line">URL = <span class="string">&#x27;https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg&#x27;</span></span><br><span class="line">response = requests.get(URL)</span><br><span class="line">I = Image.<span class="built_in">open</span>(BytesIO(response.content))</span><br><span class="line">I = I.resize([<span class="number">150</span>,<span class="number">150</span>])</span><br><span class="line">arr = np.asarray(I)</span><br><span class="line">im = PIL.Image.fromarray(np.uint8(arr))</span><br><span class="line">Image.Image.show(im)</span><br></pre></td></tr></table></figure>
+<h2 id="section-6">61-70</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 去除数组中所有的缺失值</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,np.nan,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,np.nan])</span><br><span class="line">a[~np.isnan(a)]</span><br><span class="line">array([ <span class="number">1.</span>,  <span class="number">2.</span>,  <span class="number">3.</span>,  <span class="number">5.</span>,  <span class="number">6.</span>,  <span class="number">7.</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算两个数组间的欧几里得</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>,<span class="number">5</span>])</span><br><span class="line">b = np.array([<span class="number">4</span>,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,<span class="number">8</span>])</span><br><span class="line">dist = np.linalg.norm(a-b)</span><br><span class="line">dist</span><br><span class="line"><span class="comment"># 6.7082039324993694</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中最大的两个数</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">3</span>, <span class="number">7</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">6</span>, <span class="number">0</span>, <span class="number">1</span>])</span><br><span class="line">doublediff = np.diff(np.sign(np.diff(a)))</span><br><span class="line">peak_locations = np.where(doublediff == -<span class="number">2</span>)[<span class="number">0</span>] + <span class="number">1</span></span><br><span class="line">peak_locations</span><br><span class="line"><span class="comment"># array([2, 5])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组数组数值每项减去一维数组对应的行号</span></span><br><span class="line">a_2d = np.array([[<span class="number">3</span>,<span class="number">3</span>,<span class="number">3</span>],[<span class="number">4</span>,<span class="number">4</span>,<span class="number">4</span>],[<span class="number">5</span>,<span class="number">5</span>,<span class="number">5</span>]])</span><br><span class="line">b_1d = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">a_2d - b_1d[:,<span class="literal">None</span>]</span><br><span class="line"><span class="comment"># [[2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组中重复的第n次的数的索引</span></span><br><span class="line">x = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>])</span><br><span class="line">n = <span class="number">5</span></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">[i <span class="keyword">for</span> i, v <span class="keyword">in</span> <span class="built_in">enumerate</span>(x) <span class="keyword">if</span> v == <span class="number">1</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 方法二：numpy方法</span></span><br><span class="line">np.where(x == <span class="number">1</span>)[<span class="number">0</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 8</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将`datetime64`格式转换为`datetime`</span></span><br><span class="line">dt64 = np.datetime64(<span class="string">&#x27;2018-02-25 22:10:10&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一：利用datetime模块</span></span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line">dt64.tolist()</span><br><span class="line"><span class="comment"># 方法二：numpy直接转换</span></span><br><span class="line">dt64.astype(datetime)</span><br><span class="line">datetime.datetime(<span class="number">2018</span>, <span class="number">2</span>, <span class="number">25</span>, <span class="number">22</span>, <span class="number">10</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这个没看懂。。</span></span><br><span class="line"><span class="comment"># 如何计算numpy数组的移动平均值(累加和)</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">Z = np.random.randint(<span class="number">10</span>, size=<span class="number">10</span>)</span><br><span class="line">Z</span><br><span class="line"><span class="comment"># [8 8 3 7 7 0 4 2 5 2]</span></span><br><span class="line"><span class="comment"># 方法一: np.cumsum</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">moving_average</span>(<span class="params">a, n=<span class="number">3</span></span>) :</span><br><span class="line">    ret = np.cumsum(a, dtype=<span class="built_in">float</span>)</span><br><span class="line">    ret[n:] = ret[n:] - ret[:-n]</span><br><span class="line">    <span class="keyword">return</span> ret[n - <span class="number">1</span>:] / n</span><br><span class="line">moving_average(Z, n=<span class="number">3</span>).<span class="built_in">round</span>(<span class="number">2</span>)</span><br><span class="line"><span class="comment"># 方法二: np.convolve</span></span><br><span class="line">np.convolve(Z, np.ones(<span class="number">3</span>)/<span class="number">3</span>, mode=<span class="string">&#x27;valid&#x27;</span>)</span><br><span class="line"><span class="comment"># [ 6.33  6.    5.67  4.67  3.67  2.    3.67  3.  ]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个数组，只给出起始值、步长、数组长度</span></span><br><span class="line">length = <span class="number">10</span></span><br><span class="line">start = <span class="number">5</span></span><br><span class="line">step = <span class="number">3</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seq</span>(<span class="params">start, length, step</span>):</span><br><span class="line">    end = start + (step*length)</span><br><span class="line">    <span class="keyword">return</span> np.arange(start, end, step)</span><br><span class="line">seq(start, length, step)</span><br><span class="line"><span class="comment"># array([ 5,  8, 11, 14, 17, 20, 23, 26, 29, 32])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy数组填补不完整的日期</span></span><br><span class="line">dates = np.arange(np.datetime64(<span class="string">&#x27;2018-02-01&#x27;</span>), np.datetime64(<span class="string">&#x27;2018-02-25&#x27;</span>), <span class="number">2</span>)</span><br><span class="line">dates</span><br><span class="line"><span class="comment"># [&#x27;2018-02-01&#x27; &#x27;2018-02-03&#x27; &#x27;2018-02-05&#x27; &#x27;2018-02-07&#x27; &#x27;2018-02-09&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-11&#x27; &#x27;2018-02-13&#x27; &#x27;2018-02-15&#x27; &#x27;2018-02-17&#x27; &#x27;2018-02-19&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-21&#x27; &#x27;2018-02-23&#x27;]</span></span><br><span class="line"><span class="comment"># 方法一: 列表解析</span></span><br><span class="line">filled_in = np.array([np.arange(date, (date+d)) <span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates))]).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line"><span class="comment"># 方法二: 不用列表解析循环</span></span><br><span class="line">out = []</span><br><span class="line"><span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates)):</span><br><span class="line">    out.append(np.arange(date, (date+d)))</span><br><span class="line">filled_in = np.array(out).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># array([&#x27;2018-02-01&#x27;, &#x27;2018-02-02&#x27;, &#x27;2018-02-03&#x27;, &#x27;2018-02-04&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-05&#x27;, &#x27;2018-02-06&#x27;, &#x27;2018-02-07&#x27;, &#x27;2018-02-08&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-09&#x27;, &#x27;2018-02-10&#x27;, &#x27;2018-02-11&#x27;, &#x27;2018-02-12&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-13&#x27;, &#x27;2018-02-14&#x27;, &#x27;2018-02-15&#x27;, &#x27;2018-02-16&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-17&#x27;, &#x27;2018-02-18&#x27;, &#x27;2018-02-19&#x27;, &#x27;2018-02-20&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-21&#x27;, &#x27;2018-02-22&#x27;, &#x27;2018-02-23&#x27;], dtype=&#x27;datetime64[D]&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一维数组，变形为二维数组（给定每维长度）</span></span><br><span class="line">arr = np.arange(<span class="number">15</span>)</span><br><span class="line">arr</span><br><span class="line">array([ <span class="number">0</span>,  <span class="number">1</span>,  <span class="number">2</span>,  <span class="number">3</span>,  <span class="number">4</span>,  <span class="number">5</span>,  <span class="number">6</span>,  <span class="number">7</span>,  <span class="number">8</span>,  <span class="number">9</span>, <span class="number">10</span>, <span class="number">11</span>, <span class="number">12</span>, <span class="number">13</span>, <span class="number">14</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gen_strides</span>(<span class="params">arr, stride_len=<span class="number">5</span>, window_len=<span class="number">5</span></span>):</span><br><span class="line">    n_strides = (a.size-window_len)//stride_len + <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> np.array([a[s:(s+window_len)] <span class="keyword">for</span> s <span class="keyword">in</span> np.arange(<span class="number">0</span>, n_strides*stride_len, stride_len)])</span><br><span class="line">gen_strides(np.arange(<span class="number">15</span>), stride_len=<span class="number">2</span>, window_len=<span class="number">4</span>)</span><br><span class="line"><span class="comment"># [[ 0  1  2  3]</span></span><br><span class="line"><span class="comment">#  [ 2  3  4  5]</span></span><br><span class="line"><span class="comment">#  [ 4  5  6  7]</span></span><br><span class="line"><span class="comment">#  [ 6  7  8  9]</span></span><br><span class="line"><span class="comment">#  [ 8  9 10 11]</span></span><br><span class="line"><span class="comment">#  [10 11 12 13]]</span></span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>Numpy</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>优化pandas内存入门</title>
+    <url>/hexo_blog/2018/03/17/PythonPackages/%E4%BC%98%E5%8C%96pandas%E5%86%85%E5%AD%98%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>降低数据存储消耗并提高处理效率，本文教你如何通过优化数值类型、<code>object</code>类型和
+<code>datetime</code>类型三个方面来减少内存使用量。其中，使用
+<code>Categoricals</code>优化
+<code>object</code>类型数据是效果最显著的一种优化方式。另外，还介绍了如何在读取数据的时候就完成优化步骤，避免加载完数据再做优化的麻烦。</p>
+<span id="more"></span>
+<h2 id="简单的概念">简单的概念</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"></span><br><span class="line"><span class="comment"># 读取数据</span></span><br><span class="line">df = pd.read_csv(<span class="string">&quot;test.csv&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 得到精确的内存信息</span></span><br><span class="line">df.info(memory_usage=<span class="string">&#x27;deep&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 说明：之后都用df来表示读取到的dataframe</span></span><br></pre></td></tr></table></figure>
+<p>pandas中每一个数据类型都有一个专门的类来处理。</p>
+<ul>
+<li><code>ObjectBlock</code>： 字符串列的块</li>
+<li><code>FloatBlock</code>： 浮点数列的块</li>
+<li><code>Numpy ndarray</code>：整型和浮点数值的块（非常快，用C数组构建的）</li>
+</ul>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> dtype <span class="keyword">in</span> [<span class="string">&#x27;float&#x27;</span>, <span class="string">&#x27;int&#x27;</span>, <span class="string">&#x27;object&#x27;</span>]:</span><br><span class="line">    <span class="comment"># 选中对应的dtype列</span></span><br><span class="line">    selected_dtype = df.select_dtypes(include=[dtype])</span><br><span class="line">    <span class="comment"># 查看内存使用量的平均值</span></span><br><span class="line">    mean_usage_b = selected_dtype.memory_usage(deep=<span class="literal">True</span>).mean()</span><br><span class="line">    <span class="comment"># 获取到的数据单位为K, 这里转换一下</span></span><br><span class="line">    mean_usage_mb = mean_usage_b / <span class="number">1024</span> ** <span class="number">2</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;Average memory usage for &#123;&#125; columns:&#123;:03.2f&#125; MB&quot;</span>.<span class="built_in">format</span>(dtype, mean_usage_mb))</span><br></pre></td></tr></table></figure>
+<p>这里我们可以发现 <code>object</code>类占用内存最大</p>
+<h3 id="子类型subtype">子类型(subtype)</h3>
 <table>
 <thead>
 <tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
+<th>memory usage</th>
+<th>float</th>
+<th>int</th>
+<th>uint</th>
+<th>datetime</th>
+<th>bool</th>
+<th>object</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>0</td>
-<td>0.5136</td>
-<td>0.2909</td>
-</tr>
-<tr>
-<td>1</td>
-<td>0.6143</td>
-<td>0.3322</td>
-</tr>
-<tr>
-<td>2</td>
-<td>0.7300</td>
-<td>0.3608</td>
-</tr>
-</tbody>
-</table>
-<h3 id="其他实现">其他实现</h3>
-<h4
-id="二维矩阵转一维矩阵实现矩阵乘法">二维矩阵转一维矩阵，实现矩阵乘法</h4>
-<p>在某些情况下，为了降低空间复杂度，会把二维矩阵展开为一维矩阵，再进行矩阵乘法。这里在这个基础上，再实现了
-softmax
-的计算。其本质上是把二维矩阵转为一维矩阵，再进行矩阵乘法。原来的<code>input1[row][k]</code>
--&gt; <code>input1[row * K + k]</code>，<code>input2[k][col]</code>
--&gt; <code>input2[k * N + col]</code>。但这里暂时不好实现 softmax
-融合，因为 softmax
-需要计算每行的最大值，这里的一维矩阵无法直接计算每行的最大值。所以这里偷懒先使用判断的方法，如果是最后一列，则计算
-softmax。因此，导致算的速度会比较慢。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line">#DEFINE BLOCK_SIZE <span class="number">256</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_vector_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N, <span class="type">const</span> <span class="type">int</span> K</span></span></span><br><span class="line"><span class="params"><span class="function">                                       )</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line">    <span class="type">int</span> col = index % N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M &amp;&amp; col &lt; N) &#123;</span><br><span class="line">        <span class="type">float</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; K; ++k) &#123;</span><br><span class="line">            value += input1[row * K + k] * input2[k * N + col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row * N + col] = value;</span><br><span class="line">        <span class="keyword">if</span> (col == N - <span class="number">1</span>) &#123;</span><br><span class="line">            <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">            <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                old_row_max = row_max;</span><br><span class="line">                row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">                normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">            &#125;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">matmul_vector</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> &#123;output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;), output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;)&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">matmul_vector</span>(q, k);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
-<table>
-<thead>
-<tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>0</td>
-<td>0.4239</td>
-<td>0.4907</td>
-</tr>
-<tr>
-<td>1</td>
-<td>0.5069</td>
-<td>0.4388</td>
-</tr>
-<tr>
-<td>2</td>
-<td>0.5462</td>
-<td>0.5799</td>
+<td>1 bytes</td>
+<td></td>
+<td>int8</td>
+<td>uint8</td>
+<td></td>
+<td>bool</td>
+<td></td>
 </tr>
-</tbody>
-</table>
-<p>尽管用了一种比较笨的方法，但是速度实际上与原生的 pytorch
-相差无几。</p>
-<h4 id="单独实现-softmax">单独实现 softmax</h4>
-<p>相较于非算子融合的写法，这里实现了一维矩阵的
-softmax。相当于在计算矩阵乘法后再算 softmax。比较朴素的写法。。。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">softmax_kernel</span><span class="params">(torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                 <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M) &#123;</span><br><span class="line">        <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">        <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">        <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            old_row_max = row_max;</span><br><span class="line">            row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">            normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_softmax_vector_softmax</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE_VECTOR)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 普通一维的矩阵乘法</span></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="built_in">cudaDeviceSynchronize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(output_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;);</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
-<table>
-<thead>
 <tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
+<td>2 bytes</td>
+<td>float16</td>
+<td>int16</td>
+<td>uint16</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td>0</td>
-<td>0.4239</td>
-<td>1.1874</td>
+<td>4 bytes</td>
+<td>float32</td>
+<td>int32</td>
+<td>uint32</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>1</td>
-<td>0.5069</td>
-<td>1.1101</td>
+<td>8 bytes</td>
+<td>float64</td>
+<td>int64</td>
+<td>uint64</td>
+<td>datetime64</td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>2</td>
-<td>0.5462</td>
-<td>1.3238</td>
+<td>variable</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td>object</td>
 </tr>
 </tbody>
 </table>
-<p>对比发现，这里的cuda还是会比pytorch的慢一些，这是因为没有用自带的softmax，且没有使用算子融合。所以就算速度会更慢。</p>
-<h2 id="参考">参考</h2>
-<p>[1] <a
-href="https://pytorch.org/tutorials/advanced/cpp_extension.html">https://pytorch.org/tutorials/advanced/cpp_extension.html</a>。</p>
-<p>[2] <a
-href="https://github.com/ELS-RD/kernl/blob/main/tutorial/1%20-%20tiled%20matmul.ipynb">Tiled
-matmul</a></p>
-<p>[3] <a
-href="https://github.com/ELS-RD/kernl/blob/main/tutorial/3%20-%20online%20softmax.ipynb">Online
-softmax</a></p>
-<p>[4] <a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
-calculation for softmax</a></p>
-]]></content>
-      <categories>
-        <category>DeepLearning</category>
-      </categories>
-      <tags>
-        <tag>CUDA</tag>
-        <tag>PyTorch</tag>
-        <tag>Cpp</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>torch中的scheduler</title>
-    <url>/hexo_blog/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>PyTorch学习率调整过程中版本问题引发的不同结果解析</p>
-<span id="more"></span>
-<p>代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">optimizer1 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler1 = torch.optim.lr_scheduler.MultiStepLR(optimizer1,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line">optimizer2 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler2 = torch.optim.lr_scheduler.MultiStepLR(optimizer2,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">15</span>):</span><br><span class="line">    exp_lr_scheduler1.step()</span><br><span class="line">    exp_lr_scheduler2.step(epoch)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;Epoch &#123;&#125;, lr1 &#123;&#125;, lr2 &#123;&#125;&#x27;</span>.<span class="built_in">format</span>(epoch,</span><br><span class="line">        optimizer1.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>],</span><br><span class="line">        optimizer2.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>]))</span><br></pre></td></tr></table></figure>
-<p>当torch版本为1.2.0时，输出如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
-<p>当torch版本为1.4.0时，输出如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
-<p>小结：<code>scheduler.step()</code>中的
-<code>epoch</code>参数由于版本问题会带来不同的作用效果。1.2.0版本是判断epoch是否在某个区间内，而1.4.0版本是会直接调整学习到达最后一个区间。</p>
+<p>一个
+<code>int8</code>类型值使用1个字节的存储空间，可以表示256(2^8)个二进制数，即-128到127的所有整数值。根据表可以得出，对于每个列，应该尽可能"抠"一点，
+物尽其用。这里的
+<code>uint</code>表示无符号整型，可以更有效地处理正整数的列。</p>
+<h2 id="优化">优化</h2>
+<h3 id="数值型数据">数值型数据</h3>
+<p>这里可以使用 <code>to_numeric()</code>对数值类型进行
+<code>downcast</code>（向下转型）的操作。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中整数型的列</span></span><br><span class="line">df_int_columns = df.select_dtypes(include=[<span class="string">&#x27;int&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_int_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;unsigned&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中浮点型数据</span></span><br><span class="line">df_float_columns = df.select_dtypes(include=[<span class="string">&#x27;float&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_float_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;float&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<p>这里能够有效的利用内存，但是缺点很明显的就是只能用于数值型数据，而且优化的空间有限。</p>
+<h3 id="object型数据"><code>object</code>型数据</h3>
+<p><code>object</code>类型实际上使用的是Python字符串对应的值，由于Python解释性语言的特性，字符串存储方式很碎片化（消耗更多内存，访问速度更慢）。<code>object</code>列中的每个元素都是一个指针。<code>object</code>类型的数据内存使用情况是可变的。如果每个字符串是单独存储的，那么实际上字符串占用内存是很大。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">In [<span class="number">1</span>]: <span class="keyword">from</span> sys <span class="keyword">import</span> getsizeof</span><br><span class="line"><span class="comment"># 字符串本身内存使用情况</span></span><br><span class="line">In [<span class="number">2</span>]: s1 = <span class="string">&quot;hello world!&quot;</span></span><br><span class="line">In [<span class="number">3</span>]: s2 = <span class="string">&quot;hello pandas!&quot;</span></span><br><span class="line">In [<span class="number">4</span>]: getsizeof(s1), getsizeof(s2)</span><br><span class="line">Out[<span class="number">4</span>]: (<span class="number">61</span>, <span class="number">62</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串(object)在pandas中内存使用情况</span></span><br><span class="line">In [<span class="number">9</span>]: <span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line">In [<span class="number">10</span>]: ser = pd.Series([s1, s2])</span><br><span class="line">In [<span class="number">11</span>]: ser.apply(getsizeof)</span><br><span class="line">Out[<span class="number">11</span>]:</span><br><span class="line"><span class="number">0</span>    <span class="number">61</span></span><br><span class="line"><span class="number">1</span>    <span class="number">62</span></span><br><span class="line">dtype: int64</span><br></pre></td></tr></table></figure>
+<p>这里通过观察看到，在这里pandas使用了int64来存储，实际占用大小与字符串本身是一样的。这里可以使用
+<code>Categoricals</code>来优化
+<code>object</code>类型。<code>Categoricals</code>的工作原理我理解为，某个
+<code>object</code>类有有限的分类情况(比如只有
+<code>red</code>、<code>yellow</code>、<code>blue</code>、<code>black</code>等颜色相关)，那么
+<code>Categoricals</code>将这些分类 <code>object</code>对象转换为
+<code>int</code>子类型（对应上面的0, 1, 2, 3）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中object类型</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="comment"># 查看object类型列的相关信息</span></span><br><span class="line">df_obj_columns.describe()</span><br></pre></td></tr></table></figure>
+<p>如果
+<code>unique</code>(不同值)的数量很少，那么就可以使用这种优化方案。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设某一列符合优化条件，为df_obj_less，使类型转换为category</span></span><br><span class="line">df_category_column = df_obj_less.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"><span class="comment"># 转换之后，主观观察数据没有什么区别，只是类型改变了</span></span><br><span class="line"><span class="comment"># 观察转换之后实际上的数据</span></span><br><span class="line">df_category_column.cat.codes</span><br><span class="line"><span class="comment"># 变成了由0,1,2等数字构成的int8类型的数据</span></span><br></pre></td></tr></table></figure>
+<p>这里提升的空间远远超过第一步优化的空间，具体要结合数据来检验（最好貌似可以减少98%的使用量）。这里有个很大的缺点就是无法进行数值计算，即没有办法使用
+<code>pd.Series.min()</code>、<code>pd.Series.max()</code>等与数值相关操作。</p>
+<p>还有一个问题就是在有多少个
+<code>unique</code>(不同值)的情况下才使用这种方法。首先，毫无疑问的是如果需要计算操作的列是不能使用的。第二是如果
+<code>unique</code>的比例小于50%(个人觉得比例应该更小)就可以使用这种情况，如果过多的话，转换之后消耗的内存会更多（不仅需要存储string还有int）。</p>
+<p>附上筛选 <code>unique</code>比例小于50%的代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取unique少于50%的object列</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="keyword">for</span> obj_col <span class="keyword">in</span> df_obj_columns:</span><br><span class="line">    unique_values = <span class="built_in">len</span>(df_obj_columns[obj_col].unique())</span><br><span class="line">    total_values = <span class="built_in">len</span>(df_obj_columns[obj_col])</span><br><span class="line">    <span class="keyword">if</span> unique_values / total_values &lt; <span class="number">0.5</span>:</span><br><span class="line">        obj_col.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建议提前copy一份数据，再做上面的操作</span></span><br></pre></td></tr></table></figure>
+<h3 id="datetime类型"><code>datetime</code>类型</h3>
+<p>这里其实不能叫做优化，因为这里是将数值型数据转换为
+<code>datetime</code>类型，虽然提高了内存的使用，但是转换为
+<code>datetime</code>类型的数据更容易进行分析（时间序列分析）。当然如果有
+<code>datetime</code>类型不在分析的范围内，自然可以无视。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设df_num_col为需要转换的列, 这里format格式看情况更改</span></span><br><span class="line">df_num_col = pd.to_datetime(df_num_col, <span class="built_in">format</span>=<span class="string">&quot;%Y%m%d&quot;</span>)</span><br></pre></td></tr></table></figure>
+<h2 id="总结">总结</h2>
+<p>写了这么多，如果每次等加载完数据再做优化操作，感觉有些鸡肋。但是其实可以使用
+<code>read_csv</code>等读取函数的几个参数来帮助我们在读取的时候就完成优化步骤。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先，需要整理出每一列最终的数据类型，组成一个dict</span></span><br><span class="line">dtypes_lst = [col.name <span class="keyword">for</span> name <span class="keyword">in</span> df.dtypes]</span><br><span class="line">column_types = <span class="built_in">dict</span>(<span class="built_in">zip</span>(dtypes.index, dtypes_lst))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重新读取数据, dtype指定数据格式，parse_dates指定列转换格式为datetime,infer_datetime_format尝试解析字符串形式的datatime格式（在一些情况下可以加快解析速度5-10倍）</span></span><br><span class="line">new_df = pd.read_csv(<span class="string">&#x27;test.csv&#x27;</span>, dtpye=column_types, parse_dates=[<span class="string">&#x27;date&#x27;</span>], infer_datetime_format=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
         <category>Note</category>
       </categories>
       <tags>
-        <tag>PyTorch</tag>
-        <tag>scheduler</tag>
+        <tag>Python</tag>
+        <tag>pandas</tag>
       </tags>
   </entry>
   <entry>
-    <title>交叉熵的反向传播 Step by Step（PyTorch）</title>
-    <url>/hexo_blog/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</url>
-    <content><![CDATA[<p>本文从公式开始，一步步用PyTorch实现自定义的交叉熵损失函数，最后理解分布式损失函数</p>
+    <title>用 Cpp 写 PyTorch 的插件</title>
+    <url>/hexo_blog/2023/12/03/PyTorch/pytorch_cpp_extension/</url>
+    <content><![CDATA[<p>从零开始，用 Cpp 写 PyTorch 的插件，包括 CPU 和 GPU 的版本。</p>
+<p><a
+href="https://github.com/wnma3mz/pytorch_cuda_extension">代码</a></p>
 <span id="more"></span>
-<h2 id="step-1a-simple-case">Step 1：A Simple Case</h2>
-<p>以交叉熵为例，普通的二分类问题 <span class="math display">\[
-\text {Binary Cross Entropy} = -\left (y \log (\hat {y}) + (1-y) \log
-(1-\hat {y})\right)
-\]</span> 更常见的多分类问题 <span class="math display">\[
-\text {Multiclass Cross Entropy} = -\sum_{c=0}^{C} y_c \log (\hat {y}_c)
-\]</span> 其中，<span class="math inline">\(y\)</span>
-表示真实的标签值（通常是one-hot向量），<span
-class="math inline">\(\hat{y}\)</span> 表示模型的输出，<span
-class="math inline">\(C\)</span> 表示有多少类别需要分类</p>
-<p>在 PyTorch 中调用（本文使用的torch version==2.0.0）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">criterion = torch.nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])   <span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])                 <span class="comment"># y       Shape: bs</span></span><br><span class="line"></span><br><span class="line">loss = criterion(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
-<p>这里的<code>outputs</code>是模型的输出，<code>targets</code>是真实的标签值</p>
-<h2 id="step-2-手写">Step 2: 手写</h2>
-<p>实现<code>CrossEntropyLoss</code>需要两个条件</p>
+<h2 id="为什么">为什么</h2>
+<p>一般来说，在原生功能不能满足需求的时候，插件可以作为补充。比如，PyTorch
+的 <code>torch.nn.functional</code> 中没有 <code>softmax</code>
+函数，但是 <code>torch.nn</code> 中有，所以可以用
+<code>torch.nn.functional.softmax</code> 来代替。但是，如果要用
+<code>softmax</code> 的导数，就需要用到 <code>softmax</code>
+的原始定义，这个时候就需要自己写插件了。</p>
+<p>如果是比较简单的需求，则可以直接用 Python
+完成。然而，当对性能要求较高时，往往会使用 Cpp
+来写插件，最后甚至会优化为 CUDA 代码。</p>
+<h2 id="怎么写">怎么写</h2>
+<p>从例子出发，一步步来写。假设要实现一个最简单的 Attention 模块，输入为
+<span class="math inline">\(q,k,v \in \mathbb{R}^{M \times
+N}\)</span>，输出为 <span class="math inline">\(out \in \mathbb{R}^{M
+\times N}\)</span>（不考虑 Batch Size 以及 Head 数量的情况）。Attention
+模块的计算公式为：</p>
+<p><span class="math display">\[
+out = \text{softmax}(qk^T)v
+\]</span></p>
+<p>只实现 <code>forward</code> 函数，不实现 <code>backward</code>
+函数。</p>
+<h3 id="cpu-版本">CPU 版本</h3>
+<p>类似于写 Python 的库，创建一个文件夹，目录结构如下所示：</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">attention</span><br><span class="line">├── attention.cpp</span><br><span class="line">├── setup.py</span><br><span class="line">└── __init__.py</span><br></pre></td></tr></table></figure>
+<h4 id="python-部分">Python 部分</h4>
+<p>核心代码在 <code>attention.cpp</code> 中 ，首先在
+<code>setup.py</code> 中添加如下代码：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CppExtension</span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CppExtension(<span class="string">&#x27;attention&#x27;</span>, [<span class="string">&#x27;attention.cpp&#x27;</span>]),</span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
+<p>这样，就可以用 <code>python setup.py install</code>
+来安装插件了。或者用 <code>pip install -e attention</code>
+以便于快速调试</p>
+<p>在 <code>__init__.py</code> 中导入 <code>attention</code> 模块，以在
+Python 中调用 <code>forward</code> 函数，直接计算 attention。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> .attention <span class="keyword">import</span> forward</span><br></pre></td></tr></table></figure>
+<h4 id="cpp-部分">Cpp 部分</h4>
+<p>接下来，我们需要在 <code>attention.cpp</code> 中实现
+<code>forward</code> 函数，为进行区分，这里使用这个函数名称
+<code>attention_forward</code> ，这个函数的输入是 q、k、v 三个
+<code>Tensor</code>，输出是
+<code>torch::Tensor</code>。而具体的计算步骤可以拆解为三个步骤：</p>
 <ol type="1">
-<li>使用 PyTorch 写出损失函数的公式，以及对应的导数</li>
-<li>根据公式计算loss，并计算梯度（grad）进行反向传播</li>
+<li>矩阵的乘法</li>
+<li>softmax</li>
+<li>矩阵的乘法</li>
 </ol>
-<h3 id="公式">公式</h3>
-<p>从<span class="math inline">\(-\sum_{c=0}^{C} y_c \log (\hat
-{y}_c)\)</span>出发</p>
-<ol type="1">
-<li>由于模型输出并不是单纯的概率值，因此会对模型输出进行<code>softmax</code>计算，得到<span
-class="math inline">\(\hat{y}\)</span> <span
-class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></li>
-<li>再以简单的单分类任务来说，
+<p>使用 <code>PYBIND11_MODULE</code>把
+<code>attention_forward</code>函数暴露出去，绑定到
+<code>forward</code>上，这样就能用 <code>forward</code>函数来调用
+<code>attention_forward</code>。整合后的完整代码如下，</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="comment">// 返回：方便起见，返回一个 vector，实际上只有一个元素，便于后续扩展</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = torch::<span class="built_in">matmul</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>));</span><br><span class="line">    scores = torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>);</span><br><span class="line">    torch::Tensor attention = torch::<span class="built_in">matmul</span>(scores, v);</span><br><span class="line">    <span class="keyword">return</span> &#123;attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CPU)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="测试">测试</h4>
+<p>用 Python 版本的实现来测试 Cpp 版本的实现是否正确，测试代码如下，</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> attention</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">torch.manual_seed(<span class="number">42</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_attention</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>) @ v</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_attention(q, k, v)</span><br><span class="line">    cpp_values = attention.forward(q, k, v)[-<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">q, k, v, loop=<span class="number">100</span></span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_attention(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: attention.forward(q, k, v), number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span> <span class="keyword">if</span> torch.cuda.is_available() <span class="keyword">else</span> <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;q&quot;</span>, q)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;k&quot;</span>, k)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;v&quot;</span>, v)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">20</span>)</span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    compare_time(q, k, v)</span><br></pre></td></tr></table></figure>
+<p>测试通过后，可以再使用 <code>compare_time</code>
+函数对比一下二者的速度。理论上，二者的速度是相差无几的。因为均用的是
+PyTorch 的矩阵乘法和 softmax 函数。</p>
+<p>但是，如果需要进行更进一步的优化技巧，那么就需要自己实现矩阵乘法和
+softmax 函数了。这里，我们只实现最简单的矩阵乘法和 softmax
+函数，然后再对比一下二者的速度。</p>
+<h4 id="矩阵乘法">矩阵乘法</h4>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="softmax">softmax</h4>
+<p>由于
+<code>softmax</code>函数比较特殊，后续会结合算子融合一起优化，所以简单的对其进行展开。用
+<code>torch::exp</code>和
+<code>torch::sum</code>实现了一遍，为了方便也可以直接使用
+<code>torch::softmax(scores, 1)</code></p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_softmax</span><span class="params">(<span class="type">const</span> torch::Tensor&amp; scores)</span> </span>&#123;</span><br><span class="line">    torch::Tensor exponents = torch::<span class="built_in">exp</span>(scores);</span><br><span class="line">    torch::Tensor sum = torch::<span class="built_in">sum</span>(exponents, <span class="number">1</span>, <span class="literal">true</span>);</span><br><span class="line">    <span class="keyword">return</span> exponents / sum;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>但把这两个函数替换到 <code>attention_forward</code>函数中后，再次运行
+<code>compare_time</code>函数，发现手写的 Cpp 版本的实现要比 Python
+版本的实现慢。为什么？因为，当前只是简单的实现了矩阵乘法和 softmax，而
+PyTorch 中的矩阵乘法和 softmax 都是经过优化的，所以速度会更快。</p>
+<p>另外，使用原生的矩阵乘法和 softmax 函数，可以在 GPU
+上运行，而手写的矩阵乘法和 softmax 函数，只能在 CPU
+上运行。因此，接下来将其改造为 GPU 版本，然后再进行优化。</p>
+<h3 id="gpu-版本">GPU 版本</h3>
+<h4 id="python-部分-1">Python 部分</h4>
+<p>在 <code>setup.py</code> 中更改为如下代码，把
+<code>CppExtension</code> 改为 <code>CUDAExtension</code></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CUDAExtension </span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CUDAExtension(<span class="string">&#x27;attention&#x27;</span>, [</span><br><span class="line">            <span class="string">&#x27;attention.cpp&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;attention_kernel.cu&#x27;</span>,</span><br><span class="line">        ])      </span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="cpp-部分-1">Cpp 部分</h4>
+<p>为了兼容之前的代码，这里将之前的 <code>attention_forward</code>更新为
+<code>attention_cpu_forward</code>，同时加了一个类型判断，如果输入的
+<code>Tensor</code>不在同一个设备上，则抛出异常。而对于
+<code>attention_cuda_forward</code>的实现需要在
+<code>attention_kernel.cu</code>中实现。注意：这里需要提前定义好
+<code>attention_cuda_forward</code>函数，否则会报错。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span></span>;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cpu_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line">    torch::Tensor scores = <span class="built_in">my_matmul</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">my_matmul</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;v)</span> </span>&#123;</span><br><span class="line">    <span class="keyword">if</span> (!(q.<span class="built_in">device</span>().<span class="built_in">type</span>() == k.<span class="built_in">device</span>().<span class="built_in">type</span>() &amp;&amp; q.<span class="built_in">device</span>().<span class="built_in">type</span>() == v.<span class="built_in">device</span>().<span class="built_in">type</span>())) &#123;</span><br><span class="line">        <span class="keyword">throw</span> std::<span class="built_in">runtime_error</span>(<span class="string">&quot;Input tensors q, k, and v must be on the same device&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (q.<span class="built_in">is_cuda</span>()) &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cuda_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cpu_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CUDA)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="cu-部分">Cu 部分</h4>
+<p>在同级目录下，创建 <code>attention_kernel.cu</code>文件。首先实现
+<code>attention_cuda_forward</code>函数的主要逻辑。其中，主要对矩阵乘法进行了优化，使用了
+CUDA 的并行计算。然后，使用
+<code>AT_DISPATCH_FLOATING_TYPES</code>宏来实现对不同类型的支持，这样就可以支持
+<code>float</code>和 <code>double</code>类型了。</p>
+<p>对于
+<code>matrix_multiply</code>函数，与一般写法不同的是，需要提前创建好输出的
+<code>Tensor</code>，然后再传入到 CUDA 的函数中。并且需要创建好
+<code>blocks</code>和 <code>threads</code>，然后再调用 CUDA
+的函数。这里指定了每个 CUDA 是有 16 x 16
+线程的块，而这些块是可以并行计算的，所以能够加速计算。可参考 <a
+href="https://devblogs.nvidia.com/even-easier-introduction-cuda">An Even
+Easier Introduction to CUDA</a></p>
+<p>而在传递参数的时候，需要使用 <code>packed_accessor</code>。这里的
+<code>packed_accessor</code>的第一个参数是
+<code>Tensor</code>的类型，第二个参数是
+<code>Tensor</code>的维度，第三个参数是
+<code>Tensor</code>的类型，第四个参数是
+<code>Tensor</code>的维度。这里的
+<code>packed_accessor</code>的第三个参数和第四个参数，是为了支持 CUDA
+的。</p>
+<p>接下来就是实现
+<code>matrix_multiply_kernel</code>。矩阵的乘法中，如果要计算输出矩阵的第一个值，则需要用到输入矩阵的第一行和第一列。因此，这里需要根据
+<code>block</code>和
+<code>thread</code>的索引，来计算出对应的行和列。然后，就是普通的矩阵乘法的实现了。原来的矩阵乘法的实现是：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">out = [[<span class="number">0</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(m)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(m):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(p):</span><br><span class="line">            out[i][j] += input1[i][k] * input2[k][j]</span><br></pre></td></tr></table></figure>
+<p>相当于把外面两个循环分别交给了 CUDA 的 <code>block</code>和
+<code>thread</code>来计算。这样，就可以实现并行计算了。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row][col] = value;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">matrix_multiply</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>测试代码不变，依旧可以用上面的来进行检验。至此，最简单的实验就完成了。接下来，就是对其进行优化了。</p>
+<h2 id="矩阵乘法的优化">矩阵乘法的优化</h2>
+<h3 id="matmul">Matmul</h3>
+<p>重新回到矩阵乘法上，假设有两个矩阵 A1 和 A2，形状分别为 <span
+class="math inline">\(M \times N\)</span> 和 <span
+class="math inline">\(N \times K\)</span>，则矩阵乘法的计算公式为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">output = torch.zeros(size=(M, K))</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        sum_ = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(N):</span><br><span class="line">            sum_ += A1[i][k] * A2[k][j]</span><br><span class="line">        output[i][j] = sum_</span><br></pre></td></tr></table></figure>
+<p>一种朴素的优化手段是把最后一个循环并行计算。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        output[i][j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br></pre></td></tr></table></figure>
+<p>利用多线程/进程（下文统称为job）进行并行计算可以提高程序的计算速度，但这样需要每个job都能访问到
+A1 和 A2 的数据，所以这就引入了全局内存和共享内存的概念。</p>
+<blockquote>
+<ul>
+<li>全局内存（Global
+Memory）：全局内存是一种在计算机程序中可被所有线程或进程访问的内存空间。它通常用于存储全局变量、静态变量以及动态分配的内存等。全局内存的特点是可以在整个程序执行过程中进行读写操作，但它的访问速度相对较慢。</li>
+<li>共享内存（Shared
+Memory）：共享内存是一种特殊的内存区域，被多个线程或进程同时访问和共享。通过将数据存储在共享内存中，不同的线程或进程可以直接读取和写入这些数据，而无需使用其他的通信机制。共享内存的特点是高效的数据共享和访问速度，因为不需要进行复制或传输数据。</li>
+</ul>
+</blockquote>
+<p>CUDA 中依旧存在类似的概念</p>
+<blockquote>
+<ul>
+<li>全局内存（Global Memory）：在 CUDA
+中，全局内存是一个设备（GPU）上可见的主机（CPU）内存空间。它可以由所有的线程块和线程访问，用于存储全局变量和动态分配的内存等。全局内存的读写操作相对较慢，因为涉及主机与设备之间的数据传输。</li>
+<li>共享内存（Shared Memory）：在 CUDA
+中，共享内存是位于每个线程块中的一块高速缓存内存。它被同一个线程块内的线程共享，并且比全局内存具有更快的读写速度。共享内存通常用于优化算法的性能，通过在线程块内部共享数据来减少全局内存的访问。</li>
+</ul>
+</blockquote>
+<p>简而言之，全局内存可以很方便的存各种东西，但是速度慢；共享内存是一个好东西，速度块，但通常大小受限。所以，分别有两种优化：</p>
+<ul>
+<li>全局内存中，考虑如何加速访问</li>
+<li>共享内存中，考虑如何减小占用空间</li>
+</ul>
+<p>这就引入了 Tiled matmul 算法。</p>
+<h3 id="tiled-matmul">Tiled Matmul</h3>
+<ul>
+<li>对于加速访问，在无法控制硬件的前提下，只能通过并行的方式同时读取数据。</li>
+<li>对于减小占用空间，可以通过拆分矩阵，把大矩阵拆分成若干小矩阵，然后再进行计算。</li>
+</ul>
+<p>重新思考矩阵
+<code>output</code>的计算过程，每个元素的计算其实是独立的，其本质可以拆成若干独立的小块，如下图所示：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/setup-2d0f22ecd2e9b7c84af56792d14ba18a.gif?raw=true" />
+From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
+<p>由于矩阵
+<code>output</code>每个元素是完全独立的，可以将其拆成若干个小矩阵来计算。如上图所示，把矩阵
+<code>output</code> 拆成了 4 个小矩阵。对应的代码如下所示：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/block_matrix.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size = <span class="number">2</span> <span class="comment"># 方便起见，这里设置为 N,M,K 的公约数。同时也拆成了 2x2 的 block</span></span><br><span class="line">block_M, block_N, block_K = M // block_size, N // block_size, K // block_size</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line">output11 = matmul(A1[:block_M, :], A2[:, :block_K])</span><br><span class="line">output12 = matmul(A1[:block_M, :], A2[:, block_K:])</span><br><span class="line">output21 = matmul(A1[block_M:, :], A2[:, :block_K])</span><br><span class="line">output22 = matmul(A1[block_M:, :], A2[:, block_K:])</span><br><span class="line">output = torch.cat([torch.cat([output11, output12], dim=<span class="number">1</span>), torch.cat([output21, output22], dim=<span class="number">1</span>)], dim=<span class="number">0</span>)</span><br><span class="line"><span class="built_in">print</span>(output)</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(output, A1 @ A2)</span><br></pre></td></tr></table></figure>
+<p>对于，左上角矩阵 <code>output11</code>，实际上是由
+<code>block_size</code>个矩阵乘法，再求和得到的
+<code>output11 = matmul(A1[:block_M, :block_N], A2[:block_N, :block_K]) + matmul(A1[:block_M, block_N:], A2[block_N:, :block_K])</code>。再把它扩展的灵活一点</p>
 <ol type="1">
-<li><span class="math inline">\(y\)</span> = [1, 0, 0], <span
-class="math inline">\(\hat{y}\)</span> = [0.5, 0.2, 0.3] -&gt; log(<span
-class="math inline">\(\hat{y}\)</span>)=[-0.9398, -1.2398, -1.1398]</li>
-<li>计算时，只有c=0时，结果有值（-0.9398），其他类别时为0，无需计算</li>
-<li>因此，实际最终结果就是取出对应类别的 <span
-class="math inline">\(\hat{y}\)</span> 的log值</li>
-</ol></li>
+<li>不局限于只能扩展为 2 <span class="math inline">\(\times\)</span> 2
+矩阵</li>
+<li>block_size 可以针对 M, N, K 进行调整</li>
 </ol>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># outputs.shape: bs, C</span></span><br><span class="line">    <span class="comment"># targets.shape: bs</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  <span class="comment"># 对预测结果进行softmax操作</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 使用索引选择对应类别的概率值，并使用负对数函数计算损失</span></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) <span class="comment"># 对每个计算取对应的target</span></span><br><span class="line">    <span class="keyword">return</span> loss.mean() <span class="comment"># 最终返回结果的平均值</span></span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])	<span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])	</span><br><span class="line"></span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
-<h4 id="梯度">梯度</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                <span class="comment"># 计算每个 block 的矩阵乘法</span></span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                <span class="comment"># 把每个 block 的结果放到对应的位置</span></span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"><span class="built_in">print</span>(matmul(A1, A2))</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(matmul(A1, A2), A1 @ A2)</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/tmm-59dd890f48435e692c47919d0df4a5e6.gif" />
+From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
+<p>再次回顾这样做的目的，<a
+href="https://penny-xu.github.io/blog/tiled-matrix-multiplication">原作者</a>是这么说的：</p>
+<blockquote>
+<ul>
+<li>With or without tiling, the same number of accesses into global
+memory occur. The difference is that, without tiling, each thread must
+sequentially (one after the other) access global memory 8 times.</li>
+<li>With tiling, we can parallelize the access to global memory so that
+each thread only sequentially accesses global memory 4 times.</li>
+<li>To summarize, the point is not to reduce the number of
+multiplications or even the total number of global memory accesses, but
+rather to reduce the number of sequential global memory accesses per
+thread. In other words, we better share the heavy load of memory access
+across threads.</li>
+</ul>
+</blockquote>
+<p>简而言之，作者的观点是通过拆成 block
+的形式，能够并行的读取全局内存数据。个人观点，在某些情况下，拆分后的
+block 可以恰好把数据放到共享内存中，以加速计算？</p>
+<p>最后，对于 <code>block size</code>大小的选择：</p>
+<ul>
+<li>越大的
+<code>block size</code>表示拆分后的矩阵个数变少，这样访问全局内存的次数会更多。但是，每个矩阵比较大，这样线程的并行度会更高。即，IO变小，计算变快。</li>
+<li>越小的
+<code>block size</code>表示拆分后的矩阵个数变多，这样访问全局内存的次数会更少。但是，每个矩阵比较小，这样线程的并行度会更低。即，IO变大，计算变慢。</li>
+</ul>
+<h2 id="算子融合">算子融合</h2>
+<p>算子融合是将多个计算操作合并为一个计算操作，以减少计算量和内存访问次数，从而提高计算效率。比如，矩阵乘法和
+softmax 的融合，可以减少一次内存访问。</p>
+<h3 id="softmax-的改造">softmax 的改造</h3>
+<p>softmax 的计算公式为：</p>
+<p><span class="math display">\[
+\text{softmax}(x_i) = \frac{e^{x_i}}{\sum_{j=1}^n e^{x_j}}
+\]</span></p>
+<p>而通常为了数值稳定性（避免溢出），会先计算最大值，再减去最大值，最后再计算
+softmax。即：</p>
+<p><span class="math display">\[
+\text{softmax}(x_i) = \frac{e^{x_i - \max(x)}}{\sum_{j=1}^n e^{x_j -
+\max(x)}}
+\]</span></p>
+<p>在神经网络中，其值通常为一个矩阵（不只是一个维度），所以需要对每一行进行
+softmax。示例代码如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_softmax</span>(<span class="params">X, dim=<span class="number">1</span></span>):</span><br><span class="line">    X -= torch.<span class="built_in">max</span>(X, dim=dim, keepdim=<span class="literal">True</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> torch.exp(X) / torch.<span class="built_in">sum</span>(torch.exp(X), dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(my_softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), my_softmax(X, dim=<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
+<p>但这样跟矩阵乘法进行算子融合是没有优势的，两个函数还是独立计算的。算子融合是需要糅合两种计算操作。所以，需要把
+softmax
+函数放到矩阵乘法中进行计算。即将计算过程变成一个可迭代的过程，换而言之，随着元素的增加不断更新
+softmax 的结果。</p>
+<p><a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
+calculation for softmax</a>
+提供了这么一种方式。简单起见，从一个元素开始介绍。现在有一个列表，里面只有一个元素
+<code>[1]</code>，其 softmax 计算过程为：</p>
 <ol type="1">
-<li><p>结合前面的计算，每次计算loss时，实际上只有对应类别的计算是有效的，因此将其简化为<span
-class="math inline">\(-y_c\log(\hat{y}_c)\)</span>，而由于<span
-class="math inline">\(y_c=1\)</span>，所以其实只有<span
-class="math inline">\(-\log(\hat{y}_c)\)</span></p></li>
-<li><p>这个公式只需要计算log的导数。对于<span
-class="math inline">\(f(x)=\log(x)\)</span>，其导数<span
-class="math inline">\(f^{&#39;}(x)=\frac{1}{x}\)</span>。</p></li>
-<li><p>因此，这里的梯度很简单，如下所示，其中，<span
-class="math inline">\(y_c=1\)</span> <span class="math display">\[
-\frac {\partial \text {CrossEntropy}(y, \hat {y})}{\partial \hat {y}_c}
-= -\frac {y_c}{\hat {y}_c}=-\frac{1}{\hat{y}_c}
-\]</span></p></li>
-<li><p>而对于模型的输出而言，还有一个SoftMax计算。SoftMax的梯度有</p>
+<li>exp: 计算 <span class="math inline">\(e^{1}\)</span>，得到 <span
+class="math inline">\([e^{1}]\)</span></li>
+<li>max: 计算 <code>max(1)</code>，得到 <code>1</code></li>
+<li>-max: 计算 $e^{1 - 1} $，得到 <span class="math inline">\([e^{1 - 1}
+]\)</span></li>
+<li>sum: 计算 $e^{1- 1} $ 的和，得到 $e^{1- 1} $</li>
+<li>softmax: 计算 <span class="math inline">\([e^{1- 1} ] / (e^{1- 1}
+)\)</span>，得到 <span class="math inline">\([1]\)</span></li>
+</ol>
+<p>现在增加一个元素，观察有哪些变化。假设增加的元素为
+2，现在有一个列表，里面有两个元素 <code>[1, 2]</code>，新增的 softmax
+计算过程为：</p>
 <ol type="1">
-<li>当 <span class="math inline">\(i = j\)</span> 时：</li>
+<li>exp: 计算 <span class="math inline">\(e^{2}\)</span>，得到 <span
+class="math inline">\(e^{2}\)</span></li>
+<li>max: 与原来的 <code>max(1)=1</code> 比较，得到
+<code>max(1, 2)</code>，得到 <code>2</code></li>
+<li>-max：所有元素更新一遍，均减去最新的 <code>max(1, 2)=2</code>，得到
+<span class="math inline">\([e^{1- 2} , e^{2- 2} ]\)</span></li>
+<li>sum: 重新计算一遍结果</li>
+<li>softmax: 计算 <span class="math inline">\([e^{1- 2}, e^{2 - 2}] /
+(e^{1 - 2} + e^{2 - 2})\)</span></li>
 </ol>
+<p>对比发现，第 3 步会导致重新第 4
+步计算一遍求和结果，但这个求和结果在第 5
+步中作为分母是可以灵活调整的。原来是<span
+class="math inline">\(e^{1-1}\)</span>，现在更新为<span
+class="math inline">\(e^{1-2}+e^{2-2}\)</span>。假设原来的最大值为
+<code>old_max</code>，新的最大值为 <code>new_max</code>，原来的元素为
+<code>old_v</code>，则原来元素的 <code>exp</code>结果可以更新为</p>
 <p><span class="math display">\[
-\frac {\partial y_i}{\partial x_i} = \frac {e^{x_i}\left (\sum_{k=1}^n
-e^{x_k}\right) - e^{x_i} e^{x_i}}{\left (\sum_{k=1}^n e^{x_k}\right)^2}
-= y_i (1-y_i)
+e^{old\_v-old\_max} \rightarrow e^{old\_v-old\_max+old\_max-new\_max} =
+e^{old\_v-old\_max} \times e^{old\_max-new\_max}
 \]</span></p>
-<ol start="2" type="1">
-<li>当 <span class="math inline">\(i \neq j\)</span> 时：</li>
-</ol>
+<p>而对于新加的元素，则可以在求和部分直接加上<span
+class="math inline">\(e^{2-2}\)</span>，所以，代码如下</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">nums = [<span class="number">1</span>, <span class="number">2</span>]</span><br><span class="line">sum_, max_v = <span class="number">0.</span>, <span class="number">0.</span></span><br><span class="line">norm_v = <span class="number">0.</span></span><br><span class="line"><span class="keyword">for</span> i, num <span class="keyword">in</span> <span class="built_in">enumerate</span>(nums):</span><br><span class="line">    old_max_v = max_v</span><br><span class="line">    max_v = <span class="built_in">max</span>(max_v, num)</span><br><span class="line">    norm_v = norm_v*np.exp(old_max_v - max_v) + np.exp(num - max_v)</span><br><span class="line">softmax_nums = [np.exp(num-max_v)/ norm_v <span class="keyword">for</span> num <span class="keyword">in</span> nums] </span><br></pre></td></tr></table></figure>
+<p>对于两个元素这个公式是适用的，那么对于多个元素呢？答案自然也是同样适用的</p>
 <p><span class="math display">\[
-\frac {\partial y_i}{\partial x_j} = -\frac {e^{x_j} e^{x_i}}{\left
-(\sum_{k=1}^n e^{x_k}\right)^2} = -y_i y_j
-\]</span></p></li>
-<li><p>由于求导的链式法则，因此将SoftMax的导数乘以交叉熵的导数，就能得到模型输出的梯度。综上，对于模型的输出，交叉熵的梯度仅在目标标签上有效（≠0），所以最后的梯度计算为，</p></li>
-</ol>
-<p><span class="math display">\[-\frac{1}{\hat{y}_c}\times
-y_c(1-y_c)=y_c-1\]</span></p>
-<h3 id="autograd">autograd</h3>
-<p>在PyTorch内部有一套完善的求导机制。因此，仅需按照其规则进行书写即可实现前向传播、反向传播（链式求导）。如下所示，在forward填写loss的计算，backward中填写grad的计算即可</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">CrossEntropyLoss</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        <span class="comment"># outputs：模型的输出</span></span><br><span class="line">        <span class="comment"># targets: 真实标签</span></span><br><span class="line">        ...</span><br><span class="line">        ctx.save_for_backward(...) <span class="comment"># 需要在反向传播中使用的变量</span></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        <span class="comment"># grad_output: 反向传播的梯度</span></span><br><span class="line">        ... = ctx.saved_tensors</span><br><span class="line">        ...</span><br><span class="line">	    <span class="keyword">return</span> new_grad_output</span><br></pre></td></tr></table></figure>
-<h4 id="forward">Forward</h4>
-<p>这里基本就是把公式照抄过来，但需要额外把梯度计算时用到的变量存一下。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">    ctx.save_for_backward(outputs, targets)</span><br><span class="line">    <span class="keyword">return</span> loss.mean()</span><br></pre></td></tr></table></figure>
-<h4 id="backward">Backward</h4>
-<p>这里的<code>grad_output</code>
-表示梯度计算的输出。它是一个与前向传播函数的输出形状相同的张量。在反向传播过程中，我们将计算当前函数的导数，乘以
-<code>grad_output</code>
-作为输入。这个值代表了后续节点对当前节点的梯度贡献。</p>
-<p>通过将 <code>grad_output</code>
-乘以导数（也称为雅可比向量积），可以有效地传递梯度信息到较早的节点，从而实现自动微分。</p>
-<p>注意，<code>grad_output</code>
-的形状必须与函数的输出形状一致，否则会引发错误。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">    outputs, targets = ctx.saved_tensors</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    </span><br><span class="line">    grad_y_pred = outputs.clone()</span><br><span class="line">    grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 计算对应类别的梯度</span></span><br><span class="line">    grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">    </span><br><span class="line">    grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">    </span><br><span class="line">    <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br></pre></td></tr></table></figure>
-<h3 id="完整版">完整版</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyCrossEntropyLossFunction</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">        loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">        ctx.save_for_backward(outputs, targets)</span><br><span class="line">        <span class="keyword">return</span> loss.mean()</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        outputs, targets = ctx.saved_tensors</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        </span><br><span class="line">        grad_y_pred = outputs.clone()</span><br><span class="line">        grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 只需计算对应标签的梯度</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">        </span><br><span class="line">        <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br><span class="line">  </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="keyword">return</span> MyCrossEntropyLossFunction.apply(outputs, targets)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]], requires_grad=<span class="literal">True</span>)</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])</span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398, grad_fn=&lt;MyCrossEntropyLossFunctionBackward&gt;)</span></span><br></pre></td></tr></table></figure>
-<p>结合简单的双层MLP</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MLP</span>(torch.nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>().__init__()</span><br><span class="line">        <span class="variable language_">self</span>.l1 = torch.nn.Linear(<span class="number">5</span>, <span class="number">5</span>, bias=<span class="literal">False</span>)</span><br><span class="line">        <span class="variable language_">self</span>.l2 = torch.nn.Linear(<span class="number">5</span>, <span class="number">3</span>, bias=<span class="literal">False</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.l2(<span class="variable language_">self</span>.l1(x))</span><br><span class="line">    </span><br><span class="line">input_ = torch.rand((<span class="number">5</span>, <span class="number">5</span>))</span><br><span class="line">labels = torch.randint(<span class="number">0</span>, <span class="number">3</span>, size=(<span class="number">5</span>,))</span><br><span class="line">model1 = MLP()</span><br><span class="line">model2 = copy.deepcopy(model1)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;Before Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br><span class="line">criterion1 = torch.nn.CrossEntropyLoss()</span><br><span class="line">criterion2 = MyCrossEntropyLoss</span><br><span class="line"></span><br><span class="line">optim1 = torch.optim.Adam(model1.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line">optim2 = torch.optim.Adam(model2.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line"></span><br><span class="line">outputs1 = model1(copy.deepcopy(input_))</span><br><span class="line">loss1 = criterion1(outputs1, labels)</span><br><span class="line">optim1.zero_grad()</span><br><span class="line">loss1.backward()</span><br><span class="line">optim1.step()</span><br><span class="line"></span><br><span class="line">outputs2 = model2(copy.deepcopy(input_))</span><br><span class="line">loss2 = criterion2(outputs2, labels)</span><br><span class="line">optim2.zero_grad()</span><br><span class="line">loss2.backward()</span><br><span class="line">optim2.step()</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;After Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h2 id="step-3-分布式">Step 3: 分布式</h2>
-<p>模型训练速度提升+单卡放不下大模型，由于这两个需求，所以分布式计算中需要重写损失函数的forward和backward，以Megatron-LM的训练代码为例，如下所示。接下来将逐步拆解实现过程（忽略label_smoothing，将其看作0.0）。</p>
-<p><a
-href="https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/cross_entropy.py">cross_entropy.py</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> megatron.core.parallel_state <span class="keyword">import</span> (</span><br><span class="line">    get_tensor_model_parallel_group,</span><br><span class="line">    get_tensor_model_parallel_rank,</span><br><span class="line">    get_tensor_model_parallel_world_size,</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> .utils <span class="keyword">import</span> VocabUtility</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">_VocabParallelCrossEntropy</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Maximum value along vocab dimension across all GPUs.</span></span><br><span class="line">        logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">        )</span><br><span class="line">        <span class="comment"># Subtract the maximum value.</span></span><br><span class="line">        vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">        get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">        partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">        rank = get_tensor_model_parallel_rank()</span><br><span class="line">        world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">        vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">        target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">        masked_target = target.clone() - vocab_start_index</span><br><span class="line">        masked_target[target_mask] = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get predicted-logits = logits[target].</span></span><br><span class="line">        <span class="comment"># For Simplicity, we convert logits to a 2-D tensor with size</span></span><br><span class="line">        <span class="comment"># [*, partition-vocab-size] and target to a 1-D tensor of size [*].</span></span><br><span class="line">        logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">        masked_target_1d = masked_target.view(-<span class="number">1</span>)</span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">        predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">        predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br><span class="line">        predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">        predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">        <span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            predicted_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Sum of exponential of logits along vocab dimension across all GPUs.</span></span><br><span class="line">        exp_logits = vocab_parallel_logits</span><br><span class="line">        torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line">        sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            sum_exp_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Loss = log(sum(exp(logits))) - predicted-logit.</span></span><br><span class="line">        loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Normalize and optionally smooth logits</span></span><br><span class="line">        exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        vocab_size = exp_logits.size(-<span class="number">1</span>)</span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            We&#x27;d like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + alpha * mean(y_&#123;i for i != gt&#125;)</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = (K * (1 - alpha) - 1) / (K - 1)) * y_gt  + (alpha / (K - 1)) * \sum_&#123;i&#125; y_i</span></span><br><span class="line"><span class="string">            = (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_&#123;i&#125; y_i / K</span></span><br><span class="line"><span class="string">            From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py</span></span><br><span class="line"><span class="string">            &quot;&quot;&quot;</span></span><br><span class="line">            <span class="keyword">assert</span> <span class="number">1.0</span> &gt; label_smoothing &gt; <span class="number">0.0</span></span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.</span></span><br><span class="line">            log_probs = torch.log(exp_logits)</span><br><span class="line">            mean_log_probs = log_probs.mean(dim=-<span class="number">1</span>)</span><br><span class="line">            loss = (<span class="number">1.0</span> - smoothing) * loss - smoothing * mean_log_probs</span><br><span class="line"></span><br><span class="line">        ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Store softmax, target-mask and masked-target for backward pass.</span></span><br><span class="line">        ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Retreive tensors from the forward path.</span></span><br><span class="line">        softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">        label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># All the inputs have softmax as thier gradient.</span></span><br><span class="line">        grad_input = softmax</span><br><span class="line">        <span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">        partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">        grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">        softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= (<span class="number">1.0</span> - smoothing) * softmax_update</span><br><span class="line">            average_grad = <span class="number">1</span> / vocab_size</span><br><span class="line">            grad_2d[arange_1d, :] -= smoothing * average_grad</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Finally elementwise multiplication with the output gradients.</span></span><br><span class="line">        grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> grad_input, <span class="literal">None</span>, <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">vocab_parallel_cross_entropy</span>(<span class="params">vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">    Performs cross entropy loss when logits are split across tensor parallel ranks</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Arguments:</span></span><br><span class="line"><span class="string">        vocab_parallel_logits: logits split across tensor parallel ranks</span></span><br><span class="line"><span class="string">                               dimension is [sequence_length, batch_size, hidden_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        target: correct vocab ids of dimseion [sequence_length, micro_batch_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        lobal_smoothing: smoothing factor, must be in range [0.0, 1.0)</span></span><br><span class="line"><span class="string">                         default is no smoothing (=0.0)</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">return</span> _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing)</span><br></pre></td></tr></table></figure>
-<h3 id="为什么">为什么</h3>
-<p>本文不介绍分布式训练的其他原理，仅关注模型最后的输出，以及损失函数的计算。</p>
-<p>前文已经详细介绍了交叉熵的实现，那么为什么到分布式训练的时候，就不能直接复用呢？主要原因在于模型的输出被拆散了。回顾模型的输出，其shape为[<code>bs</code>,
-<code>C</code>]，<code>bs</code>表示每次训练的数据量<code>batch size</code>，<code>C</code>表示任务的总的分类数量，在LLM中主要是词表大小<code>vocab_size</code>（下文用此代表类别数量）。</p>
-<p>对于模型的输出而言，分布式（假设有<code>word_size</code>张GPU）主要分两种情况
-1.
-拆分<code>bs</code>，每张卡上有<code>bs</code>数据量，那么整合模型的最后输出应为[<code>bs</code>
-* <code>word_size</code>, <code>vocab_size</code>] 2.
-拆分<code>vocab_size</code>，单个模型的输出为[<code>bs</code>,
-<code>vocab_size</code> //
-<code>word_size</code>]。这种情况需要保证<code>vocab_size</code>能够整除<code>word_size</code></p>
-<p>因此，在进行loss计算前，需要先同步所有的模型输出，然后再进行loss计算。这里的同步，就是从所有GPU中获取模型的输出。但考虑到通信开销问题，有些操作是可以进行优化的，即无需通信全部内容再计算。重新思考损失的计算过程可以分为如下几步：</p>
-<ol type="1">
-<li>SoftMax计算：<span class="math inline">\(\exp(x) /
-\sum(\exp(x)\)</span></li>
-<li>log的计算： <span class="math inline">\(-\log[\exp(x) /
-\sum(\exp(x)]\)</span></li>
-<li>索引：根据targets索引矩阵对应位置 <span class="math inline">\(x[...,
-...]\)</span></li>
+\begin{aligned}
+e^{v1-old\_max}+e^{v2-old\_max} &amp;\rightarrow
+e^{v1-new\_max}+e^{v2-new\_max} \\ &amp;=
+e^{v1-old\_max+old\_max-new\_max}+e^{v2-old\_max+old\_max-new\_max}
+\\  \\ &amp;= e^{v1-old\_max} \times e^{old\_max-new\_max} +
+e^{v2-old\_max} \times e^{old\_max-new\_max} \\ &amp;= (e^{v1-old\_max}
+- e^{v2-old\_max}) \times e^{old\_max-new\_max}
+\end{aligned}
+\]</span></p>
+<p>所以，将其更新为二维矩阵的形式，就有</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">online_softmax</span>(<span class="params">X</span>):</span><br><span class="line">    value = torch.zeros_like(X)</span><br><span class="line">    <span class="keyword">for</span> row <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span></span><br><span class="line">        <span class="keyword">for</span> col <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">1</span>]):</span><br><span class="line">            val = X[row, col]</span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        value[row, :] = torch.exp(X[row, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> value</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(online_softmax(X))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), online_softmax(X))</span><br></pre></td></tr></table></figure>
+<h3 id="softmax-矩阵乘法">softmax + 矩阵乘法</h3>
+<p>一般的融合操作是需要在矩阵乘法后，计算结果的 softmax。即</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line">output = torch.softmax(A1 @ A2, dim=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>在结合上面的内容后，将其改造为</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(A1.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span>  </span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(A2.shape[<span class="number">1</span>]):</span><br><span class="line">            val = output[i, j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br><span class="line">          </span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        output[i, :] = torch.exp(output[i, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
+<p>更进一步，结合 Tiled matmul，将其改造为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]            </span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">tiled_matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">    block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        old_row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        normalizer_term = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 这里算完了每个block的结果，所以需要将其拆分成每个block，然后再计算softmax</span></span><br><span class="line">            <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):            </span><br><span class="line">                <span class="keyword">for</span> jj, col <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_j, end_j)):</span><br><span class="line">                    val = output[row][col]</span><br><span class="line">                    old_row_max[ii][jj] = row_max[ii][jj]</span><br><span class="line">                    row_max[ii][jj] = <span class="built_in">max</span>(old_row_max[ii][jj], val)</span><br><span class="line">                    normalizer_term[ii][jj] = normalizer_term[ii][jj] * np.exp(old_row_max[ii][jj] - row_max[ii][jj]) + np.exp(val - row_max[ii][jj])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):</span><br><span class="line">            row_max_v, _ = torch.<span class="built_in">max</span>(row_max, dim=<span class="number">1</span>)</span><br><span class="line">            <span class="comment"># 重算 sum, 代入公式 old_v*exp(old_max - new_max)</span></span><br><span class="line">            sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br><span class="line">            output[row, :] = torch.exp(output[row, :] - row_max_v[ii]) / sum_</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(tiled_matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), tiled_matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
+<p>由于这里是每次计算出来是一个 block ，所以要把 block
+拆分出每个元素，计算 block 中每行每列的最大值 row_max 以及分母
+normalizer_term。</p>
+<p>最后在计算 softmax 时，要计算所有 block 的最大值
+<code>torch.max(row_max, dim=1)</code>，还需要计算分母，这里需要考虑所有的
+block，可以类比于合并计算 <code>[1, 2]</code>,
+<code>[3, 4]</code>两个列表的normalizer_term 。已知对应的
+normalizer_term <span class="math inline">\(e^{1-2}+e^{2-2}\)</span> 和
+<span class="math inline">\(e^{3-4}+e^{3-4}\)</span>，合并后的结果应当是
+<span
+class="math inline">\(e^{1-4}+e^{2-4}+e^{3-4}+e^{3-4}\)</span>。将其公式化写作：</p>
+<p><span class="math display">\[
+\begin{aligned}
+\sum_{i=1}^{m} e^{x_i-max(x)} + \sum_{i=1}^{n} e^{y_i-max(y)} &amp;
+\rightarrow \sum_{i=1}^{m} e^{x_i-max(x,y)} + \sum_{i=1}^{n}
+e^{y_i-max(x,y)} \\ &amp;= \sum_{i=1}^{m}
+e^{x_i-max(x)}e^{max(x)-max(x,y)} + \sum_{i=1}^{n}
+e^{y_i-max(x,y)}e^{max(y)-max(x,y)}
+\end{aligned}
+\]</span></p>
+<p>假设 <code>max(x,y)=max(x)</code>，那么对应项将乘
+1，这并不会对结果有任何影响。对应的代码为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br></pre></td></tr></table></figure>
+<p>至此，完成了矩阵乘法和 softmax 的融合。接下来，会实现 cuda
+版本以对比性能。</p>
+<h3 id="cuda-实现">Cuda 实现</h3>
+<p>首先，接着之前矩阵乘法的cuda实现，</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">    <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">        value += input1[row][k] * input2[k][col];</span><br><span class="line">    &#125;</span><br><span class="line">    output[row][col] = value</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>在循环之后，已经计算完了输出矩阵中的一项。需要继续算每行的
+<code>row_max</code>和 <code>normalizer_term</code>。但由于这里是 cuda
+中的某个 block，所以需要借助共享内存来通信每行的结果。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">__shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">__shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">row_max[threadIdx.y][threadIdx.x] = value; <span class="comment">// 先把计算结果放到 row_max 中，以便于比较大小</span></span><br><span class="line">__syncthreads(); <span class="comment">// 这行代码是为了保证每个线程都已经计算完了，才能进行下一步的操作</span></span><br></pre></td></tr></table></figure>
+<p>计算过程分为三个步骤：1. 找到每行的最大值</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li>计算每行的 softmax 的分母每项组成</li>
 </ol>
-<p>其中，前两者跟最后一个步骤是可以独立运行的。而前两者的计算是耦合的，为节约通信量，对log计算进行变化</p>
-<ol type="1">
-<li><span class="math inline">\(-\log[\exp(x) /
-\sum(\exp(x)]\)</span></li>
-<li><span class="math inline">\(-\log[\exp(x)] +
-\log[\sum\exp(x)]\)</span></li>
-<li><span class="math inline">\(\log[\sum\exp(x)] - x\)</span></li>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line">normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line">__syncthreads();</span><br></pre></td></tr></table></figure>
+<ol start="3" type="1">
+<li>计算每行的 softmax 的每项之后</li>
 </ol>
-<p>因此，可以由原始的<code>func1</code>推导至分布式的<code>func2</code>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func1</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 原始版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    <span class="comment"># Step 0:</span></span><br><span class="line">    predicted_logits = outputs[<span class="built_in">range</span>(bs), targets]</span><br><span class="line"></span><br><span class="line">    predicted_logits = torch.softmax(predicted_logits, dim=<span class="number">1</span>)</span><br><span class="line">    loss = -torch.log(predicted_logits)</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func2</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 分布式的简化版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">    min_c, max_c = <span class="number">0</span>, <span class="number">3</span> <span class="comment"># 预先设定的最小和最大值</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 只选择在该设备上的数值</span></span><br><span class="line">    target_mask = (targets &lt; min_c) | (targets &gt;= max_c)</span><br><span class="line">    <span class="comment"># 由于在分布式中min_c不一定为0，所以需要做一个归一化</span></span><br><span class="line">    masked_target = targets.clone() - min_c</span><br><span class="line">    masked_target[target_mask] = min_c</span><br><span class="line"></span><br><span class="line">    <span class="comment"># arange_1d, masked_target 分别对应 上文的range(bs), targets</span></span><br><span class="line">    arange_1d = torch.arange(start=<span class="number">0</span>, end=bs)</span><br><span class="line">    predicted_logits = outputs[arange_1d, masked_target]</span><br><span class="line">    </span><br><span class="line">    predicted_logits = predicted_logits.clone().contiguous()</span><br><span class="line">    <span class="comment"># 对于非该设备上计算的值，置为0。则不进行计算</span></span><br><span class="line">    predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">    <span class="comment"># 先算第二项</span></span><br><span class="line">    predicted_logits = torch.<span class="built_in">sum</span>(predicted_logits, dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">    loss = torch.log(torch.exp(outputs).<span class="built_in">sum</span>(dim=-<span class="number">1</span>)) - predicted_logits</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])    </span><br><span class="line"></span><br><span class="line">func1(outputs, targets)</span><br><span class="line">func2(outputs, targets)</span><br></pre></td></tr></table></figure></p>
-<h3 id="forward-1">Forward</h3>
-<h4 id="预处理">预处理</h4>
-<p>一般而言，为了数值稳定性，SoftMax计算前会减去最大值，即</p>
-<p><span class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}} = \frac {e^{x_i - \max(x)}}{\sum_{j=1}^n
-e^{x_j - \max(x)}}\]</span></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每张卡上的模型输出为 vocab_parallel_logits，shape：[bs, seq_len, vocab_size // word_size]</span></span><br><span class="line"><span class="comment"># SoftMax的前置计算，可以无需通信全部参数，分两步进行</span></span><br><span class="line"><span class="comment"># 1. 计算每张卡的最大值</span></span><br><span class="line"><span class="comment"># 2. 再计算所有卡的最大值</span></span><br><span class="line"><span class="comment"># 3. 最后对每卡张减去最大值</span></span><br><span class="line"><span class="comment"># 这样就能避免通信所有的参数</span></span><br><span class="line">logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">)</span><br><span class="line"><span class="comment"># Subtract the maximum value.</span></span><br><span class="line">vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>这里的<code>seq_len</code>是LLM中句子的长度，句子的每个位置（token）会计算下一个token的概率（在词表中选），因此，可以理解为每个位置都是一个分类任务，因此<code>vocab_size</code>就是分类的数量。即，可以转换为[<code>bs</code>*<code>seq_len</code>,
-<code>vocab_size</code> // <code>word_size</code>]的形式</p>
-<p>由于单卡不一定存在所有词表的值，因此需要在这张卡上将非该卡输出位置的值置为0。
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">rank = get_tensor_model_parallel_rank()</span><br><span class="line">world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">masked_target = target.clone() - vocab_start_index</span><br><span class="line">masked_target[target_mask] = <span class="number">0</span></span><br></pre></td></tr></table></figure></p>
-<h4 id="根据-targets-选择模型真实输出的logits">根据 targets
-选择模型真实输出的Logits</h4>
-<p>将模型的输出和标签 转换为[<code>bs</code>*<code>seq_len</code>,
-<code>vocab_size</code> // <code>word_size</code>]的形式，以便于计算
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">masked_target_1d = masked_target.view(-<span class="number">1</span>) </span><br><span class="line"></span><br><span class="line"><span class="comment"># 选择真实标签的logits，相当于原来的outputs[range(bs), targets]</span></span><br><span class="line"><span class="comment"># 相当于把此步骤置前，而对于后续计算无影响，并减少了后续的通信时间</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br></pre></td></tr></table></figure></p>
-<h4 id="计算loss">计算loss</h4>
-<p>将logits转换为原来的shape，再求和，先得到了最后的第一项，之后再减去log
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line"><span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    predicted_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure></p>
-<p>计算SoftMax <span class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">exp_logits = vocab_parallel_logits</span><br><span class="line"><span class="comment"># 计算每张卡的exp，作为分子</span></span><br><span class="line">torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line"><span class="comment"># 再计算这张卡上的exp之和</span></span><br><span class="line">sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 再计算所有卡的exp之和，作为分母</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    sum_exp_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure>
-<p>计算loss <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一项是log(sum(exp(logits)))，第二项是最后的logits</span></span><br><span class="line">loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为了Backward，需要保存一些变量</span></span><br><span class="line">exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure></p>
-<h3 id="backward-1">Backward</h3>
-<h4 id="预处理-1">预处理</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据SoftMax计算的结果，忽略无需计算梯度的位置。target_mask中的元素不是0就是1</span></span><br><span class="line">grad_input = softmax</span><br><span class="line"><span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br></pre></td></tr></table></figure>
-<h4 id="梯度计算">梯度计算</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 正常计算位置，就-1，否则将不减（即不做计算）</span></span><br><span class="line">grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后乘上关于loss的梯度</span></span><br><span class="line">grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Algorithm</category>
-      </categories>
-      <tags>
-        <tag>PyTorch</tag>
-        <tag>数学</tag>
-        <tag>交叉熵</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>OpenWrt路由器折腾记录</title>
-    <url>/hexo_blog/2023/01/10/IOT/OpenWrt%E8%B7%AF%E7%94%B1%E5%99%A8%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>在路由器上安装和配置OpenWrt。它包括安装软件包和设置防火墙、接口和各种网络连接的分步说明。</p>
-<span id="more"></span>
-<h2 id="基本界面">基本界面</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353557758.png" /></p>
-<p>准备：</p>
-<ul>
-<li>固件</li>
-<li>待恢复的配置</li>
-<li>需要安装的软件+运行的命令</li>
-</ul>
-<p>配置出问题，直接重刷。同时方便起见，要及时备份当前的配置。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353799884.png" /></p>
-<h2 id="软件包的安装">软件包的安装</h2>
-<p>能在线安装的可以直接在线安装。直接从"系统-&gt;软件包"里面找到需要安装的软件。</p>
-<p>但有时候在线安装会由于网络，软件源、软件版本等问题，无法自动安装。因此需要我们手动找到ipk文件进行安装。使用下面的命令进行安装。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">opkg install XXX</span><br></pre></td></tr></table></figure>
-<p>官方软件下载网页：<a
-href="https://downloads.openwrt.org/releases/">https://downloads.openwrt.org/releases/</a></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 系统版本可以直接从管理界面的概况中得知，这里是17.01.4</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 内核版本</span></span><br><span class="line">&gt;&gt;&gt; <span class="built_in">uname</span> -a </span><br><span class="line">Linux dw33d 4.4.194 <span class="comment">#0 Tue Oct 17 17:46:20 2017 mips GNU/Linux</span></span><br><span class="line"><span class="comment"># 查看支持的架构</span></span><br><span class="line">&gt;&gt;&gt; opkg print-architecture</span><br><span class="line"><span class="built_in">arch</span> all 1</span><br><span class="line"><span class="built_in">arch</span> noarch 1</span><br><span class="line"><span class="built_in">arch</span> mips_24kc 10</span><br></pre></td></tr></table></figure>
-<p>对应我这里的第三方包就是：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/</a></p>
-<p>基础的库：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base</a></p>
-<p>luci：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/</a></p>
-<h3 id="tmux">tmux</h3>
-<p>以tmux安装为例，该软件无法直接安装。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 1. 从package里面下载tmux的ipkg文件</span></span><br><span class="line"><span class="comment"># 2. 执行安装命令</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 3. 报错信息，一些依赖文件没有。所以从base里面找到对应缺失的ipk文件。并进行安装</span></span><br><span class="line">&gt;&gt;&gt;opkg install terminfo_6.0-1_mips_24kc.ipk libncurses_6.0-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 4. 此时，再安装tmux。安装成功</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 但安装libevent2这个库时，提示当前已安装，并且版本过高，并不会自动降级。所以需要手动卸载原有的软件</span></span><br><span class="line">&gt;&gt;&gt;opkg remove libevent2 --force-depends</span><br><span class="line"><span class="comment"># 卸载失败，提示已有软件依赖该库。所以我们需要强制进行卸载，将依赖软件也进行卸载。加上--force 命令</span></span><br><span class="line"><span class="comment"># 之后再安装低版本的libncurses</span></span><br><span class="line">&gt;&gt;&gt;opkg install libevent2_2.0.22-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后再运行tmux，成功</span></span><br></pre></td></tr></table></figure>
-<p>其他软件应该同理，需要一步一步找到依赖文件，并且耐心地解决各种问题。dw33d内置了一个16g的sd卡，所以我把一些重装必备的ipk文件放到里面，便于快速安装。并且刷写固件并不会影响该sd卡的内容，可以放心存储。</p>
-<h3 id="tailscale">Tailscale</h3>
-<p>异地组网工具，从官网下载找到mips版本，放到sd卡中，避免刷新固件有影响。相当于单文件就可以运行。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压文件</span></span><br><span class="line">&gt;&gt;&gt; tar zxvf tailscale_1.34.1_mips.tgz</span><br><span class="line"><span class="comment"># 到对应的目录中，里面会有一些文件。主要是两个可执行文件tailscale和tailscaled。</span></span><br><span class="line"><span class="comment"># 第一次运行需要启动tailscaled服务</span></span><br><span class="line">&gt;&gt;&gt;./tailscaled --state=tailscaled.state</span><br><span class="line"><span class="comment"># 此时需要新开一个窗口，登录tailscale</span></span><br><span class="line">&gt;&gt;&gt;./tailscale login </span><br><span class="line"><span class="comment"># 可以直接把网址复制出来，用其他设备的浏览器进行登录</span></span><br></pre></td></tr></table></figure>
-<p>登录完毕之后，组网就完成了。此时，可以关闭该窗口。但为了后台启动，所以可以使用tmux，或者nohup命令。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
-<p>同时为了保证开机自启动，所以我们可以在web管理界面中"系统-&gt;启动项"中，最后的"本地启动脚本"中插入(<code>exit 0</code>之前行)</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先切换到对应目录，再执行</span></span><br><span class="line"><span class="built_in">cd</span> /mnt/sda1/tailscale_1.34.1_mips &amp;&amp; <span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
-<p>这样，就能保证异地管理该路由器的web界面，并且可以直接ssh登录</p>
-<h2 id="防火墙">防火墙</h2>
-<p>"网络-&gt;防火墙"中的默认配置，wan口是拒绝入站数据的。这样就会导致如果该路由器跟其他设备同属于一个局域网，ssh和web界面均不能通过ip登录。具体的例子是：</p>
-<ul>
-<li>该路由器连接了一个上级路由器A，获取到的IP是192.168.0.X。分配给下面的设备的IP是192.168.1.X</li>
-<li>另一个设备接入路由器A时，获取到的IP是192.168.0.Y。此时能ping通192.168.0.X，但是无法进行ssh，也无法通过192.168.0.X访问管理界面</li>
-<li>另一个设备接入该路由器时，获取到的IP是192.168.1.Y。此时能够通过ssh和web访问192.168.1.X。</li>
-</ul>
-<p>需求：在同局域网下的其他设备也能访问该路由器。</p>
-<p>做法：</p>
-<ul>
-<li>一个简单的做法就是如下图所示，把"防火墙"中的"常规设置"里面的wan口，入站数据从<strong>拒绝</strong>改成<strong>接受</strong>。（默认是拒绝入站的）</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1680509223.png" /></p>
-<ul>
-<li>另一个做法是配置"防火墙-&gt;端口转发"，把需要配置的端口转发出来。如下图所示，web界面的端口是80。所以从wan口出来的访问80端口的流量，直接转发到192.168.4.1（该路由器分配给下面的IP，不是获取的IP）</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356610477.png" /></p>
-<p>关于流量规则这里，则是在系统内部决定是否开放某个端口。可以参考默认的一些配置进行开放，一般应该是不需要调整的。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356804612.png" /></p>
-<h2 id="接口">接口</h2>
-<p>"网络-&gt;接口"</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357070459.png" /></p>
-<p>我主要会有以下四种上网方式（不会专业术语，简单描述现象）：</p>
-<ul>
-<li><p>桥接光猫</p>
-<ul>
-<li>将路由器WAN口接入光猫LAN口，代替光猫进行拨号，需要知道宽带的账号密码。</li>
-<li>在WAN口中，协议选择<strong>PPPoE</strong>，输入账号密码</li>
-<li>此时该路由器就是主路由器，一般同时有ipv4和ipv6两个地址，假设ipv4的ip为192.168.1.1</li>
-<li>接入该路由器的设备，由该路由器进行分配ip，ip与路由器同网段，如192.168.1.2</li>
-</ul></li>
-<li><p>桥接上一级路由器</p>
-<ul>
-<li>将路由器WAN口接入上一级路由器LAN口，无需知道账号密码</li>
-<li>在WAN口中，协议选择<strong>DHCP客户端</strong></li>
-<li>在物理设置中，选择桥接接口，默认<strong>以太网交换机:
-eth0</strong>开启。勾选两个无线网络。如下图所示</li>
-<li><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673524264094.png" /></li>
-<li>此时，IP同上一级路由器同网段，且如果有IPv6，则连接该路由器的设备也有IPv6</li>
-</ul></li>
-<li><p>做NAT</p>
-<ul>
-<li>将路由器WAN接入上一级路由器的LAN口，相当于把该路由器做一个新的入口。</li>
-<li>在WAN中，协议选择<strong>DHCP客户端</strong>或<strong>静态地址</strong>（需要额外配置）</li>
-<li>此时，接入该路由器的设备会独立上一级路由器的设备，即网段不同。</li>
-<li>注：如果此时还需要ipv6，可以参考这篇文件进行配置。<a
-href="https://www.lategege.com/?p=676">https://www.lategege.com/?p=676</a>
-<ul>
-<li>简单来说，就是需要选择静态IP，并在<strong>WAN口</strong>和<strong>LAN</strong>中的"DHCP服务器"中的IPv6设置中，路由器广告服务，DHCPv6服务，NDP-代理三个均选择中继模式</li>
-<li>并且在终端中 <code>vim /etc/config/dhcp</code>，找到
-<code>config dhcp 'wan'</code>对应的选项，加入
-<code>option master '1'</code>这一行</li>
-</ul></li>
-</ul></li>
-<li><p>WiFi放大器</p>
-<ul>
-<li><p>无需网线</p></li>
-<li><p>"网络-&gt;无线"中的搜索，找到需要放大WiFi，加入网络。看情况选择放大2.4G还是5G的WiFi</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357519480.png" /></p></li>
-<li><p>WPA密钥就是WiFi密码，新的网络名称可以默认也可以另起一个以便区分。</p></li>
-<li><p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357576678.png" /></p></li>
-<li><p>此时接口中会多一个接口</p></li>
-<li><p>其他同桥接</p></li>
-</ul></li>
-</ul>
-]]></content>
-      <categories>
-        <category>Linux</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>路由器</tag>
-        <tag>OpenWrt</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>安卓手机利用Termux安装Linux</title>
-    <url>/hexo_blog/2021/07/21/IOT/%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E5%88%A9%E7%94%A8Termux%E5%AE%89%E8%A3%85Linux/</url>
-    <content><![CDATA[<p>本文介绍如何在安卓手机上安装和运行Ubuntu系统，并通过ssh连接进行操作，同时给出了一些其他有用的命令和配置说明。</p>
-<span id="more"></span>
-<h3 id="安装">安装</h3>
-<p>不要从Google Play上安装，正确的安装包地址<a
-href="https://f-droid.org/packages/com.termux/">f-droid</a></p>
-<p>下载完成后，正常安装，但需要额外的给存储空间权限。</p>
-<p>虽然说可以免root安装，但没有root权限，有些操作不能很好的完成，如
-<code>top</code></p>
-<h3 id="连接">连接</h3>
-<p>在手机上配置太麻烦，所以可以使用同一局域网下的电脑进行配置</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 手机上</span></span><br><span class="line"><span class="comment"># 安装ssh服务</span></span><br><span class="line">pkg install openssh -y</span><br><span class="line"><span class="comment"># 开启ssh服务器</span></span><br><span class="line">sshd</span><br><span class="line"><span class="comment"># 配置密码</span></span><br><span class="line">passwd</span><br><span class="line"><span class="comment"># 输入想要配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 电脑终端, ip填写对应的手机ip，8022是termux默认的ssh端口</span></span><br><span class="line">ssh root@[ip] -p8022</span><br><span class="line"><span class="comment"># 输入刚刚的密码</span></span><br></pre></td></tr></table></figure>
-<p>至此，就能在电脑端进行配置</p>
-<h3 id="配置">配置</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装第三方库</span></span><br><span class="line">pkg install proot git python wget -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># 一键安装项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/sqlsec/termux-install-linux.git</span><br><span class="line"><span class="built_in">cd</span> termux-install-linux</span><br><span class="line">python termux-linux-install.py</span><br><span class="line"><span class="comment"># 选择Ubuntu</span></span><br><span class="line"><span class="built_in">cd</span> ~/Termux-Linux/Ubuntu</span><br><span class="line">./start-ubuntu.sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或者，ubuntu版本，最新支持20.04</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/MFDGaming/ubuntu-in-termux</span><br><span class="line"><span class="built_in">cd</span> ubuntu-in-termux</span><br><span class="line"><span class="built_in">chmod</span> +x ubuntu.sh</span><br><span class="line">./ubuntu.sh -y</span><br><span class="line">./startubuntu.sh</span><br></pre></td></tr></table></figure>
-<p>这样安装的Linux是与Termux共享文件目录的，所以可以直接将文件拷贝在
-<code>ubuntu-fs/root</code>文件夹中，启动
-<code>./startubuntu.sh</code>后，就能直接看到文件。</p>
-<p>注：若进入系统后，无法更新软件源，可能是dns存在问题，可以在termux中更改dns解析（因为系统中不好编辑）。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim ubuntu-fs/etc/resolv.conf</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">nameserver 114.114.114.114</span><br><span class="line">nameserver 8.8.8.8</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 若依旧不行，可尝试其他dns，如1.1.1.1等</span></span><br></pre></td></tr></table></figure>
-<h3 id="其他">其他</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pkg install vim tmux -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># frp项目地址，手机请注意下载arm版本，一般为arm64，不能直接在Linux系统中配置，必须得在Termux中操作</span></span><br><span class="line">[https://github.com/fatedier/frp/releases](https://github.com/fatedier/frp/releases)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机温度，需要除以1000</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/thermal/thermal_zone0/temp</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机电池容量，需root</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/power_supply/battery/capacity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置termux的启动操作</span></span><br><span class="line">vim ~/.bashrc</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">sshd <span class="comment"># 开启ssh服务，便于连接</span></span><br><span class="line"><span class="comment"># 配置frp相关操作，便于直接远程连接</span></span><br><span class="line"><span class="comment"># 配置进入Linux系统操作</span></span><br><span class="line">`</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Android</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>Android</tag>
-        <tag>Termux</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>对各种电子产品的拓展玩法与互联</title>
-    <url>/hexo_blog/2019/09/14/IOT/%E5%AF%B9%E5%90%84%E7%A7%8D%E7%94%B5%E5%AD%90%E8%AE%BE%E5%A4%87%E7%9A%84%E6%8B%93%E5%B1%95%E7%8E%A9%E6%B3%95%E4%B8%8E%E4%BA%92%E8%81%94/</url>
-    <content><![CDATA[<p>介绍手中各种电子产品及各自的拓展玩法</p>
-<span id="more"></span>
-<h2
-id="对各种电子产品的拓展玩法与互联">对各种电子产品的拓展玩法与互联</h2>
-<p>手上有的电子产品，其中大部分应该是每个人都有的。</p>
-<ul>
-<li><p>魅族手机</p></li>
-<li><p>笔记本电脑（Win10）</p></li>
-<li><p>1T的机械移动硬盘</p></li>
-<li><p>若干U盘。。</p></li>
-<li><p>Kindle Paper White 3</p></li>
-<li><p>树莓派Zero</p></li>
-<li><p>IPad 2018（IOS13前）</p></li>
-</ul>
-<h3 id="手机与ipad">手机与IPad</h3>
-<p>这两个放在一起的原因是，在IOS13之前，IPad其实差不多可以等同于大尺寸IPone。所以这里就是一个安卓手机（未ROOT）与IPone的介绍，这里的玩法也算是比较多的，下面一一介绍。关于APP的安装包在此就不提供了，请读者自行搜索。当然还有ADB这种玩法，也不做介绍，可参考
-<a href="#">Post not found: IOT/用ADB调试安卓手机.md</a></p>
-<h4 id="termux与juicessh">Termux与JuiceSSH</h4>
-<p>放在搜索引擎一搜，其实这两个APP就有很多相关介绍了。但这里我想介绍一些不一样的玩法。关于Termux操作不方便这一点，可以使用JuiceSSH连接localhost来操作，具体操作不介绍。</p>
-<p><strong>下载</strong>：如何在手机上实现多线程下载？下一个ADM？当然这样很方便，但我们同样可以使用Termux来完成这一点。Termux安装Axel（<code>pkg install axel</code>），下载<code>axel -n 32 url</code>，使用32个线程下载链接。建议先<code>cd /sdcard/</code>中。</p>
-<p><strong>数据传输</strong>：一个手机和一个IPad，如何在不借助其他介质的前提下快速传输。QQ、微信、快牙等传输文件工具都行，但这里我自己实测速度都不如这里的方法快。就拿快牙举例，它实际是用手机开放一个<code>2.4G</code>的热点，然后用二维码进行链接传输。同样的，我们用同样的思路，手机开放<code>5G</code>热点，IPad手动连接此WIFI。打开Termux，先<code>ifconfig</code>知道本地IP，形如<code>192.168.x.1</code>这种，再<code>cd</code>到传输文件的目录，利用Python的静态资源服务器<code>python3 -m http.server</code>，这样，资源就对局域网内设备（连接到手机热点的设备）开放。接下来操作IPad，由于IOS13之前，<code>Safari</code>不支持输入链接下载，所以这里推荐<code>Alook</code>与<code>Documents</code>（后者免费），在<code>Documents</code>中的浏览器输入刚刚的手机<code>ip:8000</code>，这样就能看到手机资源所在文件目录的所有文件，点击即可下载。实测单个文件速度大概比<code>2.4G</code>快3倍（无需流量）。当然这里手机和IPad也可以同时连接同一个WIFI，然后使用同上方法进行传输（速度取决于路由器和网络带宽）。</p>
-<p><strong>远程手机</strong>：如何在手机不连接WIFI，不连接数据线的前提下，用其他设备访问手机（手机与其他设备的实际距离也可以很远），前提手机能上网。这一方法个人暂没有特别用途，但<code>Just for fun</code>。首先，需要一个公网IP服务器；接下来需要会玩ssh，<code>Termux</code>与服务器配置好ssh，<code>Termux</code>能够ssh到服务器上即可。最后一步，参考
-<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>。</p>
-<h4 id="spacedesk">spacedesk</h4>
-<p>电脑投屏，使用USB数据线连接设备和电脑，用这个软件进行投屏。我觉得挺鸡肋的。。。安卓手机屏幕又太小，测试IPad的时候发现闪退。</p>
-<h4
-id="androidhttpcapture和httpcanary">AndroidHttpCapture和HttpCanary</h4>
-<p>安卓抓包工具，后者的功能比前者更强大一些。有时候身边没有电脑，需要手机抓包看数据，所以就只能直接用手机看数据包了。前者基本满足了大部分场景的抓包数据。但当某些时候（比如某些APP的数据交互不是使用HTTP等常用协议，可能就没办法抓包，此时电脑上没办法直接抓包），所以就需要使用<code>HttpCanary</code>（需要ROOT）来对网络进行代理，在解析数据。测试如果电脑上使用<code>WireShark</code>，确实是可以抓到包的，但是会乱码。所以可能使用<code>HttpCanary</code>还更为方便一些。</p>
-<h4 id="netx与net-analyzer">NetX与Net Analyzer</h4>
-<p>安卓局域网设备分析工具，用的不多，具体不介绍，有兴趣自行下载、</p>
-<h4 id="ios同类软件免费">IOS同类软件，免费</h4>
-<ul>
-<li><p><code>iTerminal</code>，同<code>JuiceSSH</code></p></li>
-<li><p><code>Stream</code>，同<code>AndroidHttpCapture</code></p></li>
-<li><p><code>Fing</code>，同<code>NetX</code></p></li>
-</ul>
-<h4 id="scrcpy">scrcpy</h4>
-<p>在电脑上使用，安卓投屏工具，<a
-href="https://github.com/Genymobile/scrcpy">scrcpy</a>，由于魅族手机一些独特的操作，所以需要替换jar文件，<a
-href="https://github.com/taie02/taie02.github.io/tree/master/2019/03/03/scrcpy_usage">此项目</a>。</p>
-<h4 id="otg">OTG</h4>
-<p>关于OTG这一项，其实是属于外接设备这一项，通过充电接口转USB接口可以接大部分usb设备，比如u盘这种。甚至还可以反向充电，接鼠标键盘等。</p>
-<h3 id="u盘与移动硬盘">U盘与移动硬盘</h3>
-<p>二者都属于移动存储介质，其实也没什么好介绍的。。。可以提几点，Windows
-To
-Go（WTG）可以搞（高性能的移动盘），自己没尝试过。在移动硬盘上装了两个Linux系统。</p>
-<p>关于如何装系统就不介绍了，这里只有一点需要注意的是备份好数据再分区，在装系统的时候引导装载移动硬盘上，要不然移动硬盘上只能在装系统的设备上启动。</p>
-<h3 id="kindle">Kindle</h3>
-<p>这里不介绍越狱之类的操作，只有常规操作。。。</p>
-<ul>
-<li><p><a
-href="https://bookfere.com/post/19.html">搭建个人推送网站</a></p></li>
-<li><p><a href="https://volmoe.com/">漫画</a></p></li>
-<li><p>Chrome插件：Push to Kindle</p></li>
-</ul>
-<h3 id="树莓派">树莓派</h3>
-<p>吃灰派。。。大概也就能跑跑小demo，爬数据，利用空闲网络带宽下东西。同样也可以利用ssh，进行端口转发，远程连接。</p>
-<p>后续再更。。。</p>
-]]></content>
-      <categories>
-        <category>Life</category>
-      </categories>
-      <tags>
-        <tag>Raspberry</tag>
-        <tag>Linux</tag>
-        <tag>Android</tag>
-        <tag>Windows</tag>
-        <tag>IOS</tag>
-        <tag>Kindle</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>用ADB调试安卓手机</title>
-    <url>/hexo_blog/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</url>
-    <content><![CDATA[<p>使用adb工具实现对安卓手机的调试，包括安装应用、查看运行App、录屏、截图等功能。本文详细介绍了adb工具的安装、配置和使用方法，并列出了一些实用的命令，同时还介绍了无线网调试设备的方法。</p>
-<span id="more"></span>
-<p>写在开头，一些推荐阅读的博客文章</p>
-<p><a
-href="http://blog.csdn.net/fasfaf454/article/details/51438743?locationNum=14">通过adb获取手机信息</a></p>
-<p><a href="https://www.jianshu.com/p/5980c8c282ef">Android
-adb你真的会用吗?</a></p>
-<p><a
-href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
-<h3 id="介绍">介绍</h3>
-<p>adb是安卓开发调试工具，接触到这个还是因为微信跳一跳的外挂原因。看文档说明之后得知需要adb工具，进行简单尝试后，发现这玩意还真的是挺好用的。填补了自己用代码控制安卓手机的空缺（之前也玩过按键精灵，但是需要Root权限）。</p>
-<p><a
-href="https://developer.android.com/studio/command-line/adb.html?hl=zh-cn"><strong>adb官方说明</strong></a>：</p>
-<p>Android 调试桥 (adb)
-是一个通用命令行工具，其允许您与模拟器实例或连接的 Android
-设备进行通信。它可为各种设备操作提供便利，如安装和调试应用，并提供对
-Unix
-shell（可用来在模拟器或连接的设备上运行各种命令）的访问。该工具作为一个客户端-服务器程序，包括三个组件：</p>
-<h3 id="安装">安装</h3>
-<p>可参考文章: <a
-href="https://www.xda-developers.com/install-adb-windows-macos-linux/">How
-to Install ADB on Windows, macOS, and Linux</a></p>
-<h4 id="windows">Windows</h4>
-<p>这里我使用的是win10。如果以下操作不能成功配置，可以百度其他的安装教程</p>
-<ol type="1">
-<li><p>下载。<a
-href="https://dl.google.com/android/repository/platform-tools-latest-windows.zip">下载链接</a></p></li>
-<li><p>移动目录</p>
-<ol type="1">
-<li>解压压缩包</li>
-<li>移动文件夹你想让它放置的位置</li>
-</ol></li>
-<li><p>环境配置</p>
-<ol type="1">
-<li>添加环境变量，这样可以在cmd或者powershell中直接使用adb</li>
-<li>右键"计算机"-&gt;"属性"-&gt;左边的"高级系统设置"-&gt;"环境变量"&gt;-在上面用户变量中的"Path"进行编辑-&gt;"添加"刚刚放置的文件夹目录，确认保存即可
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135305187.png" /></li>
-<li>这样再打开cmd或者powershell，就可以直接使用adb命令了</li>
-</ol></li>
-</ol>
-<h4 id="linux">Linux</h4>
-<p>Linux系统安装adb工具还是很容易的，我使用的是Ubuntu17.10</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> apt updagte</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt upgrade</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt install android-tools-adb</span><br></pre></td></tr></table></figure>
-<h3 id="配置">配置</h3>
-<p>手机需要打开USB调试模式，在实验的时候，市面上流行的大部分安卓手机USB调式模式开关默认关闭。当然如果手机默认开启了USB调试开关，自然可以省去下面一步。这里简单介绍一下如何手动打开USB调试模式开关，手机“设置”—&gt;"关于手机"—&gt;点击"版本号"7次。每个手机开启的方式大同小异，如果有不同，可以百度“xx手机如何开启USB调试”。</p>
-<p>在USB调试开关开启之后，进入USB调试，打开USB调试开关，如下图。</p>
-<p><img src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135319543.png" width="300px" /></p>
-<p>之后使用USB数据线连接到电脑。</p>
-<p>连接成功后，手机需要信任电脑。电脑打开cmd/powershell（Windows）或者终端（Linux），输入
-<code>adb devices</code>。查看输出。</p>
-<p>如果有输出设备名且无警告，则表示连接成功，否则需要进行下面一步的配置。</p>
-<p><strong>Windows</strong></p>
-<p>首先需要连接手机，确保手机能连接到电脑且打开了USB调试。</p>
-<p>第二步，右键"计算机"-&gt;"属性"-&gt;左边的"设备管理器"-&gt;"便携设备"-&gt;"详细信息"-&gt;"属性"下拉选择"硬件Id"，查看"值"中的内容。</p>
-<p>第三步，进入C盘用户目录下的隐藏文件夹".android"，编辑或者新建文件"adb_usb.ini"，用记事本打开，在里面写上内容。我这里需要写的是"0x2A45"，0x是前缀不需要更改，后面的字符串对应的更改"硬件Id"里面"值"的VID后面的四个字符。保存退出即可</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135348748.png" /></p>
-<p>第四步，断开USB连接，重新连接，再输入
-<code>adb devices</code>。正常情况下应该会出现设备的成功连接的信息</p>
-<p><strong>Linux</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先连接手机，输入下面命令，找到关于连接手机的那一行</span></span><br><span class="line">&gt; lsusb</span><br><span class="line"><span class="comment"># 输出形如下面，最后的&quot;Google Inc&quot;表示的是设备名，这里需要找到匹配自己手机的设备，并记录好ID</span></span><br><span class="line">...</span><br><span class="line">Bus 001 Device 017: ID 18d1:d002 Google Inc.</span><br><span class="line">...</span><br><span class="line"><span class="comment"># 在目录/etc/udev/rules.d/下，添加文件70-android.rules(名字貌似无所谓？？？)</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 写入下面内容，idVendor表示ID冒号前的数字，idProduct表示的是ID冒号后的数字，MODE固定为&quot;0666&quot;，其他不变</span></span><br><span class="line">SUBSYSTEM==<span class="string">&quot;usb&quot;</span>, ATTRS&#123;idVendor&#125;==<span class="string">&quot;18d1&quot;</span>, ATTRS&#123;idProduct&#125;==<span class="string">&quot;d002&quot;</span>,MODE=<span class="string">&quot;0666&quot;</span></span><br><span class="line"><span class="comment"># 保存退出，修改权限</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chmod</span> 644   /etc/udev/rules.d/70-android.rules</span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chown</span> root. /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 重启服务</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service udev restart</span><br><span class="line">&gt; <span class="built_in">sudo</span> killall adb</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输入adb devices查看输出，是否有误</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="使用">使用</h3>
-<p>详细可参考这篇博客: <a
-href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
-<p>在这里贴出一些我认为比较有意思且实用的功能</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看所有连接设备</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行截图保存在sd卡的根目录下，名字为screen.png</span></span><br><span class="line">&gt; adb shell screencap -p /sdcard/screen.png</span><br><span class="line"><span class="comment"># 将截图发送到本地（当前目录下），也可以发送其他文件</span></span><br><span class="line">&gt; adb pull /sdcard/screen.png</span><br><span class="line"><span class="comment"># 删除本地文件</span></span><br><span class="line">&gt; adb shell <span class="built_in">rm</span> /sdcard/screen.png</span><br><span class="line"><span class="comment"># 发送电脑里的文件到设备</span></span><br><span class="line">&gt; adb shell push screen.png /sdcard/</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入手机的交互环境，操作类似linux终端，exit或者Ctrl+C退出</span></span><br><span class="line">&gt; adb shell</span><br><span class="line"></span><br><span class="line"><span class="comment"># 点击手机屏幕(1000,1000)的位置</span></span><br><span class="line">&gt; adb shell input tap 1000 1000</span><br><span class="line"><span class="comment"># 输入字符串&quot;helloworld&quot;，此处不能直接输入中文，且字符串不能有空格</span></span><br><span class="line">&gt; adb shell input text helloworld</span><br><span class="line"><span class="comment"># 滑动屏幕，从（100, 100)到（1000,1000)，经历10s(也可以当作长按屏幕来使用)</span></span><br><span class="line">&gt; adb shell input swipe 100 100 1000 1000 10</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看当前运行的App, 这里Windows没有grep所以会运行失败，可以进入先进入交互环境再输入下面去掉&quot;adb shell&quot;命令</span></span><br><span class="line">&gt; adb shell dumpsys window | grep mCurrentFocus</span><br><span class="line"><span class="comment"># 或者</span></span><br><span class="line">&gt; adb shell dumpsys activity activities | grep mFocusedActivity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 按下电源键</span></span><br><span class="line">&gt; adb shell input keyevent 26</span><br><span class="line"><span class="comment"># 按下返回键</span></span><br><span class="line">&gt; adb shell input keyevent 4</span><br><span class="line"><span class="comment"># 按下HOME健</span></span><br><span class="line">&gt; adb shell input keyevent 3</span><br><span class="line"><span class="comment"># 点亮屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 224</span><br><span class="line"><span class="comment"># 熄灭屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 223</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机安装了哪些App，输出按行输出App的包名</span></span><br><span class="line">&gt; adb shell pm list packages</span><br><span class="line"><span class="comment"># 加&quot;-s&quot;表示只输出系统应用</span></span><br><span class="line"><span class="comment"># 加&quot;-3&quot;表示只输出第三方应用</span></span><br><span class="line"><span class="comment"># 加字符串表示过滤应用名称，当然也可以使用grep</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装apk</span></span><br><span class="line">&gt; adb install &lt;packagename&gt;</span><br><span class="line"><span class="comment"># 卸载apk</span></span><br><span class="line">&gt; adb uninstall &lt;packagename&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 从桌面启动app</span></span><br><span class="line">&gt; adb shell monkey -p &lt;packagename&gt; -c android.intent.category.LAUNCHER 1</span><br><span class="line"><span class="comment"># 关闭app</span></span><br><span class="line">&gt; adb shell am force-stop &lt;packagename&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="使用无线网调试设备">使用无线网调试设备</h3>
-<p>确保手机和电脑在同一局域网内</p>
-<p>如何查看手机的ip，由于每个设备的方法不大相同，在此不进行说明，请读者自行百度。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用USB连接设备，启动手机5555端口</span></span><br><span class="line">&gt; adb tcpip 5555</span><br><span class="line"><span class="comment"># adb连接命令，需要知道手机的ip</span></span><br><span class="line">&gt; adb connect 192.168.1.111</span><br><span class="line"><span class="comment"># 输入如下则表示连接成功</span></span><br><span class="line">connected to 192.168.1.111:5555</span><br><span class="line"><span class="comment"># 这里要注意的是，如果接下来的步骤未断掉数据线，则电脑是连接了两台设备的，进行操作会产生错误，所以可以拔掉数据线或者断开网络或者指定设备运行</span></span><br><span class="line"><span class="comment"># 断开无线网连接</span></span><br><span class="line">&gt; adb disconnect 192.168.1.111</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Android</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>ADB</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Matplotlib笔记</title>
-    <url>/hexo_blog/2017/08/26/PythonPackages/Matplotlib%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>解决Python中文乱码问题的完整方案及Matplotlib绘图基础操作</p>
-<span id="more"></span>
-<h1 id="中文乱码解决方案">中文乱码解决方案</h1>
-<p>支持所有平台的解决方案</p>
-<h2 id="定位默认字体位置">定位默认字体位置</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先进入python的交互环境</span></span><br><span class="line">&gt;&gt;&gt;python</span><br><span class="line"></span><br><span class="line">&gt;from matplotlib.font_manager import findfont, FontProperties</span><br><span class="line"><span class="comment"># 当前使用的默认字体</span></span><br><span class="line">&gt;findfont(FontProperties(family=FontProperties().get_family()))</span><br><span class="line"><span class="comment"># 在此记住返回值</span></span><br></pre></td></tr></table></figure>
-<h2 id="下载支持中文的字体">下载支持中文的字体</h2>
-<p>在这里推荐微软雅黑，msyh.ttf。</p>
-<p><a
-href="http://www.monmonkey.com/sonota/font1/getfont.html">推荐链接</a></p>
-<h2 id="偷梁换柱">偷梁换柱</h2>
-<p>在刚刚查询的那个文件夹中，将原来的默认字体重命名，之后将我们下载msyh.ttf替换进去，并重命名为那个默认字体之前的名字</p>
-<h2 id="大功告成">大功告成</h2>
-<p>当然为了保险起见，还可以操作下面一步。到现在为止，可以测试看看是否成功输出中文。</p>
-<p>找到matplotlib的安装目录，一般是在python的安装目录下的-packages-data,之后修改matplotlibrc文件</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将下面这两行冒号后面的字符替换</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#font.sans-serif     : nothing</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#verbose.level  : debug</span></span><br></pre></td></tr></table></figure>
-<h1 id="matplotlib.pyplot-基础操作">matplotlib.pyplot 基础操作</h1>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> matplotlib.pyplot <span class="keyword">as</span> plt</span><br><span class="line"></span><br><span class="line">x_list = []</span><br><span class="line">y_list = []</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定画布</span></span><br><span class="line">plt.figure(figsize=(<span class="number">8</span>, <span class="number">6</span>), dpi=<span class="number">80</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画y_list的折线图，格式为&#x27;ro-&#x27;，颜色为blue</span></span><br><span class="line">plt.plot(y_list, <span class="string">&#x27;ro-&#x27;</span>, color=<span class="string">&#x27;blue&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴画折线图，前提x_list内容全为整型数字</span></span><br><span class="line">plt.plot(x_list, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴，x_list为字符串</span></span><br><span class="line">x = <span class="built_in">range</span>(<span class="built_in">len</span>(x_list))</span><br><span class="line">plt.plot(x, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定横坐标, 跟着上面的字符串x_list的设定，每个字符串旋转角度为45，字体大小为20</span></span><br><span class="line">plt.xticks(x, x_list, rotation=<span class="number">45</span>, fontsize=<span class="number">20</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定标题，注如果需要设定中文需要更改matplotlib的字体配置</span></span><br><span class="line">plt.title(<span class="string">&#x27;title&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 保存图片名为plt</span></span><br><span class="line">plt.savefig(<span class="string">&quot;plt.png&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 运行完打开图片</span></span><br><span class="line">plt.show()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 注：如果需要保存图片，切记需要在打开图片前执行命令，否则保存图片为空白</span></span><br></pre></td></tr></table></figure>
-<h1 id="搬运工系列">搬运工系列</h1>
-<p><a
-href="https://juejin.im/post/5a9e14726fb9a028b86d87c9">5种快速易用的Python
-Matplotlib数据可视化方法</a></p>
-<p><a
-href="http://www.labri.fr/perso/nrougier/teaching/matplotlib/">Matplotlib
-教程</a></p>
-<p><a
-href="https://liam0205.me/2014/09/11/matplotlib-tutorial-zh-cn/">Matplotlib
-教程（中文翻译版本）</a></p>
-]]></content>
-      <categories>
-        <category>Visualization</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>Matplotlib</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Numpy的70个操作</title>
-    <url>/hexo_blog/2018/03/02/PythonPackages/Numpy%E7%9A%8470%E4%B8%AA%E6%93%8D%E4%BD%9C/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Numpy从入门到进阶的操作,
-翻译文章，便于自己使用时查阅</p>
-<span id="more"></span>
-<p><a
-href="https://www.machinelearningplus.com/101-numpy-exercises-python/">原文链接</a></p>
-<p>注释及输出前带有 <code>#</code></p>
-<h2 id="section">1-10</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看numpy的版本</span></span><br><span class="line">np.__version__</span><br><span class="line"><span class="comment"># 1.13.3</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个一维数据</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个布尔型数组</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.full((<span class="number">3</span>, <span class="number">3</span>), <span class="literal">True</span>, dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.ones((<span class="number">3</span>, <span class="number">3</span>), dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># array([[ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True]], dtype=bool)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数组中为奇数的数字</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([1, 3, 5, 7, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中为奇数的数字为-1</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>] = -<span class="number">1</span></span><br><span class="line"><span class="comment"># 方法二, 更加安全的操作方法（这里算新的一个操作）：</span></span><br><span class="line">out = np.where(arr % <span class="number">2</span> == <span class="number">1</span>, -<span class="number">1</span>, arr)</span><br><span class="line">out</span><br><span class="line"><span class="comment"># array([ 0, -1,  2, -1,  4, -1,  6, -1,  8, -1])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组维度变换</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line">arr.reshape(<span class="number">2</span>, -<span class="number">1</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 纵向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">0</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.vstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.r_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 横向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.hstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.c_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4, 1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9, 1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据原有的数组，自由拓展</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]) <span class="comment"># 原来的数组</span></span><br><span class="line">np.r_[np.repeat(a, <span class="number">3</span>), np.tile(a, <span class="number">3</span>)]</span><br><span class="line"><span class="comment"># array([1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-1">11-20</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.intersect1d(a, b)</span><br><span class="line"><span class="comment"># array([2, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在a数组中移去a、b数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>])	<span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>])	<span class="comment"># 原来的数组b</span></span><br><span class="line">np.setdiff1d(a, b)</span><br><span class="line"><span class="comment"># array([1, 2, 3, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共元素（索引相同）的索引</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.where(a == b)</span><br><span class="line"><span class="comment"># (array([1, 3, 5, 7], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取指定数值区间的元素</span></span><br><span class="line">a = np.arange(<span class="number">15</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">index = np.where((a &gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">index = np.where(np.logical_and(a&gt;=<span class="number">5</span>, a&lt;=<span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法三</span></span><br><span class="line">a[(a&gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>)]</span><br><span class="line"><span class="comment"># array([ 5,  6,  7,  8,  9, 10])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy运用函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">maxx</span>(<span class="params">x, y</span>):		<span class="comment"># 需要使用的函数</span></span><br><span class="line">    <span class="keyword">if</span> x &gt;= y:</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="keyword">return</span> y</span><br><span class="line"></span><br><span class="line">a = np.array([<span class="number">5</span>, <span class="number">7</span>, <span class="number">9</span>, <span class="number">8</span>, <span class="number">6</span>, <span class="number">4</span>, <span class="number">5</span>]) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">6</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">7</span>, <span class="number">1</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">pair_max = np.vectorize(maxx, dtypes=[<span class="built_in">float</span>])</span><br><span class="line">pair_max(a, b)</span><br><span class="line"><span class="comment"># array([ 6.,  7.,  9.,  8.,  9.,  7.,  5.])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两行</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[:, [<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>]]</span><br><span class="line"><span class="comment"># array([[1, 0, 2],</span></span><br><span class="line"><span class="comment">#        [4, 3, 5],</span></span><br><span class="line"><span class="comment">#        [7, 6, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两列</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[[<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>], :]</span><br><span class="line"><span class="comment"># array([[3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2],</span></span><br><span class="line"><span class="comment">#        [6, 7, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每行逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[6, 7, 8],</span></span><br><span class="line"><span class="comment">#        [3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每列逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[2, 1, 0],</span></span><br><span class="line"><span class="comment">#        [5, 4, 3],</span></span><br><span class="line"><span class="comment">#        [8, 7, 6]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个二维数组，数值区间为5-10</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">rand_arr = np.random.randint(low=<span class="number">5</span>, high=<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>)) + np.random.random((<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">rand_arr = np.random.uniform(<span class="number">5</span>,<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[ 6.75709583,  7.4662972 ,  7.2395679 ],</span></span><br><span class="line"><span class="comment">#        [ 9.7802112 ,  6.70830431,  6.76776651],</span></span><br><span class="line"><span class="comment">#        [ 6.35564151,  7.8632512 ,  8.53040631],</span></span><br><span class="line"><span class="comment">#        [ 7.99457763,  7.85526309,  8.96584465],</span></span><br><span class="line"><span class="comment">#        [ 6.18621991,  8.14214732,  7.31162037]])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-2">21-30</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出数组时，控制输出精度</span></span><br><span class="line">rand_arr = np.random.random([<span class="number">3</span>,<span class="number">3</span>]) / <span class="number">1e3</span>  <span class="comment"># 原始数组</span></span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.43404942e-04,   2.78369385e-04,   4.24517591e-04],</span></span><br><span class="line"><span class="comment">#        [  8.44776132e-04,   4.71885619e-06,   1.21569121e-04],</span></span><br><span class="line"><span class="comment">#        [  6.70749085e-04,   8.25852755e-04,   1.36706590e-04]])</span></span><br><span class="line"><span class="comment"># 第一个操作，指定小数点后3位数</span></span><br><span class="line">np.set_printoptions(precision=<span class="number">3</span>)</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.434e-04,   2.784e-04,   4.245e-04],</span></span><br><span class="line"><span class="comment">#        [  8.448e-04,   4.719e-06,   1.216e-04],</span></span><br><span class="line"><span class="comment">#        [  6.707e-04,   8.259e-04,   1.367e-04]])</span></span><br><span class="line"><span class="comment"># 第二个操作，避免输出e-04，并指定小数点后6位数</span></span><br><span class="line">np.set_printoptions(suppress=<span class="literal">True</span>, precision=<span class="number">6</span>)</span><br><span class="line"><span class="comment"># array([[ 0.000543,  0.000278,  0.000425],</span></span><br><span class="line"><span class="comment">#        [ 0.000845,  0.000005,  0.000122],</span></span><br><span class="line"><span class="comment">#        [ 0.000671,  0.000826,  0.000137]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 含有很多元素的数组</span></span><br><span class="line">a = np.arange(<span class="number">15</span>)	<span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定显示元素个数的阈值</span></span><br><span class="line">np.set_printoptions(threshold=<span class="number">6</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2, ..., 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定不隐藏元素输出</span></span><br><span class="line">np.set_printoptions(threshold=np.nan)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 导入网站数据</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">iris[:<span class="number">3</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取某一列</span></span><br><span class="line">species = np.array([row[<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris])</span><br><span class="line">species</span><br><span class="line"><span class="comment"># array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;|S15&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取多列</span></span><br><span class="line"><span class="comment"># 方法一，在导入的时候指定列</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])</span><br><span class="line"><span class="comment"># 方法二，使用上面的方法</span></span><br><span class="line">iris = np.array([row.tolist()[:<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris], dtype=<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line"><span class="comment"># array([[ 5.1,  3.5,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  3. ,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.3,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.6,  3.1,  1.5,  0.2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算平均值、方差、中位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">mu, med, sd = np.mean(sepallength), np.median(sepallength), np.std(sepallength)</span><br><span class="line"><span class="comment"># 5.84333333333 5.8 0.825301291785</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 0-1化数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">Smax, Smin = sepallength.<span class="built_in">max</span>(), sepallength.<span class="built_in">min</span>()</span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">S = (sepallength - Smin)/(Smax - Smin)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">S = (sepallength - Smin)/sepallength.ptp()</span><br><span class="line"><span class="comment"># array([ 0.222222,  0.166667,  0.111111, ...,  0.611111,  0.527778,</span></span><br><span class="line"><span class="comment">#         0.444444])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># softmax数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">softmax</span>(<span class="params">x</span>):</span><br><span class="line">    e_x = np.exp(x - np.<span class="built_in">max</span>(x))</span><br><span class="line">    <span class="keyword">return</span> e_x / e_x.<span class="built_in">sum</span>(axis=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">softmax(sepallength)</span><br><span class="line"><span class="comment"># array([ 0.00222 ,  0.001817,  0.001488, ...,  0.009001,  0.006668,  0.00494 ])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-3">31-40</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数据中百分位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="comment"># 提取数据中5%, 95%的数</span></span><br><span class="line">np.percentile(sepallength, q=[<span class="number">5</span>, <span class="number">95</span>])</span><br><span class="line"><span class="comment"># array([ 4.6  ,  7.255])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数据中的缺失值替换为随机数</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">i, j = np.where(iris_2d)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.choice((i), <span class="number">20</span>), np.random.choice((j), <span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line">iris_2d[:<span class="number">10</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.6&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.4&#x27;, b&#x27;3.9&#x27;, b&#x27;1.7&#x27;, b&#x27;0.4&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.4&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, nan, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数据中缺失值位置</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 统计缺失值个数</span></span><br><span class="line">np.isnan(iris_2d[:, <span class="number">0</span>]).<span class="built_in">sum</span>()</span><br><span class="line"><span class="comment"># 5</span></span><br><span class="line"><span class="comment"># 统计缺失值位置</span></span><br><span class="line">np.where(np.isnan(iris_2d[:, <span class="number">0</span>]))</span><br><span class="line"><span class="comment"># (array([ 38,  80, 106, 113, 121], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组多重条件筛选</span></span><br><span class="line">condition = (iris_2d[:, <span class="number">2</span>] &gt; <span class="number">1.5</span>) &amp; (iris_2d[:, <span class="number">0</span>] &lt; <span class="number">5.0</span>)</span><br><span class="line">iris_2d[condition]</span><br><span class="line"><span class="comment"># array([[ 4.8,  3.4,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.4,  1.9,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.1,  1.6,  nan],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  2.4,  3.3,  1. ]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除有缺失值的行，注意~</span></span><br><span class="line">any_nan_in_row = np.array([~np.<span class="built_in">any</span>(np.isnan(row)) <span class="keyword">for</span> row <span class="keyword">in</span> iris_2d])</span><br><span class="line"><span class="comment"># 未删除前</span></span><br><span class="line">iris_2d.shape</span><br><span class="line"><span class="comment"># (150, 4)</span></span><br><span class="line"><span class="comment"># 删除后</span></span><br><span class="line">iris_2d[any_nan_in_row].shape()</span><br><span class="line"><span class="comment"># (130, 4)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算数组间关联度</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">np.corrcoef(iris[:, <span class="number">0</span>], iris[:, <span class="number">2</span>])[<span class="number">0</span>, <span class="number">1</span>]</span><br><span class="line"><span class="comment"># 0.87175415730487182</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 检查数组中是否含有缺失值</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中的缺失值为0</span></span><br><span class="line">iris_2d[np.isnan(iris_2d)] = <span class="number">0</span></span><br><span class="line"><span class="comment"># 替换前</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"><span class="comment"># 替换后</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># False</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数组中非重复数值的情况</span></span><br><span class="line">np.unique(iris, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># 上面的数组的表示数字，下面的数组表示出现次数</span></span><br><span class="line"><span class="comment"># (array([ 0.1,  0.2,  0.3,  0.4,  0.5,  0.6,  1. ,  1.1,  1.2,  1.3,  1.4,</span></span><br><span class="line"><span class="comment">#          1.5,  1.6,  1.7,  1.8,  1.9,  2. ,  2.1,  2.2,  2.3,  2.4,  2.5,</span></span><br><span class="line"><span class="comment">#          2.6,  2.7,  2.8,  2.9,  3. ,  3.1,  3.2,  3.3,  3.4,  3.5,  3.6,</span></span><br><span class="line"><span class="comment">#          3.7,  3.8,  3.9,  4. ,  4.1,  4.2,  4.3,  4.4,  4.5,  4.6,  4.7,</span></span><br><span class="line"><span class="comment">#          4.8,  4.9,  5. ,  5.1,  5.2,  5.3,  5.4,  5.5,  5.6,  5.7,  5.8,</span></span><br><span class="line"><span class="comment">#          5.9,  6. ,  6.1,  6.2,  6.3,  6.4,  6.5,  6.6,  6.7,  6.8,  6.9,</span></span><br><span class="line"><span class="comment">#          7. ,  7.1,  7.2,  7.3,  7.4,  7.6,  7.7,  7.9]),</span></span><br><span class="line"><span class="comment">#  array([ 6, 28,  7,  7,  1,  1,  8,  4,  7, 20, 20, 26, 11,  6, 12,  7,  7,</span></span><br><span class="line"><span class="comment">#          6,  6, 12,  6, 11,  5,  9, 14, 10, 27, 12, 13,  8, 12,  8,  4,  4,</span></span><br><span class="line"><span class="comment">#          7,  5,  6,  4,  5,  3,  8,  9,  7,  7,  9, 11, 14, 17,  6,  3,  8,</span></span><br><span class="line"><span class="comment">#         10, 12, 11, 10,  5,  8,  9,  4, 10,  8,  5,  3, 10,  3,  5,  1,  1,</span></span><br><span class="line"><span class="comment">#          3,  1,  1,  1,  4,  1], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数组中处于相应区间的数换位文本</span></span><br><span class="line"><span class="comment"># 将数组中数组位于[0, 3, 5, 10]区间内的转换为对应的索引</span></span><br><span class="line">petal_length_bin = np.digitize(iris[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>), [<span class="number">0</span>, <span class="number">3</span>, <span class="number">5</span>, <span class="number">10</span>])</span><br><span class="line">label_map = &#123;<span class="number">1</span>: <span class="string">&#x27;small&#x27;</span>, <span class="number">2</span>: <span class="string">&#x27;medium&#x27;</span>, <span class="number">3</span>: <span class="string">&#x27;large&#x27;</span>, <span class="number">4</span>: np.nan&#125;</span><br><span class="line">petal_length_cat = [label_map[x] <span class="keyword">for</span> x <span class="keyword">in</span> petal_length_bin]</span><br><span class="line">petal_length_cat[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># [&#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;]</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-4">41-50</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在已经存在的数组中，多加一列</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 计算</span></span><br><span class="line">sepallength = iris_2d[:, <span class="number">0</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">petallength = iris_2d[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">volume = (np.pi * petallength * (sepallength**<span class="number">2</span>))/<span class="number">3</span></span><br><span class="line">volume = volume[:, np.newaxis]</span><br><span class="line"><span class="comment"># 横向堆叠</span></span><br><span class="line">out = np.hstack([iris_2d, volume])</span><br><span class="line">out[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 38.13265162927291],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 35.200498485922445],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 30.0723720777127],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 33.238050274980004]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对数组进行概率抽样</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">species = iris[:, <span class="number">4</span>]</span><br><span class="line"><span class="comment"># 方法一：生成概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.array([<span class="string">&#x27;Iris-setosa&#x27;</span>, <span class="string">&#x27;Iris-versicolor&#x27;</span>, <span class="string">&#x27;Iris-virginica&#x27;</span>])</span><br><span class="line">species_out = np.random.choice(a, <span class="number">150</span>, p=[<span class="number">0.5</span>, <span class="number">0.25</span>, <span class="number">0.25</span>])</span><br><span class="line"><span class="comment"># 方法二：直接概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">probs = np.r_[np.linspace(<span class="number">0</span>, <span class="number">0.500</span>, num=<span class="number">50</span>), np.linspace(<span class="number">0.501</span>, <span class="number">.750</span>, num=<span class="number">50</span>), np.linspace(<span class="number">.751</span>, <span class="number">1.0</span>, num=<span class="number">50</span>)]</span><br><span class="line">index = np.searchsorted(probs, np.random.random(<span class="number">150</span>))</span><br><span class="line">species_out = species[index]</span><br><span class="line">np.unique(species_out, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># (array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-versicolor&#x27;, b&#x27;Iris-virginica&#x27;], dtype=object), array([77, 37, 36], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对筛选后的数组，取出第二大的数</span></span><br><span class="line">petal_len_setosa = iris[iris[:, <span class="number">4</span>] == <span class="string">b&#x27;Iris-setosa&#x27;</span>, [<span class="number">2</span>]].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">np.unique(np.sort(petal_len_setosa))[-<span class="number">2</span>]</span><br><span class="line"><span class="comment"># 1.7</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对二维数组按照其中某一列进行排序</span></span><br><span class="line"><span class="comment"># 按照第一列升序</span></span><br><span class="line">iris[iris[:,<span class="number">0</span>].argsort()][:<span class="number">20</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;4.3&#x27;, b&#x27;3.0&#x27;, b&#x27;1.1&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.0&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;2.9&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.5&#x27;, b&#x27;2.3&#x27;, b&#x27;1.3&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.6&#x27;, b&#x27;1.0&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.2&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.9&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.1&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.4&#x27;, b&#x27;3.3&#x27;, b&#x27;1.0&#x27;, b&#x27;Iris-versicolor&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.5&#x27;, b&#x27;4.5&#x27;, b&#x27;1.7&#x27;, b&#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计出现次数最多的数</span></span><br><span class="line">vals, counts = np.unique(iris[:, <span class="number">3</span>], return_counts=<span class="literal">True</span>)</span><br><span class="line">vals[np.argmax(counts)]</span><br><span class="line"><span class="comment"># b&#x27;0.2&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中第一次出现比给定数字大的值的缩影</span></span><br><span class="line"><span class="comment"># 这里设定要找比1.0大的值</span></span><br><span class="line">np.argmax(iris[:, <span class="number">3</span>].astype(<span class="string">&#x27;float&#x27;</span>) &gt; <span class="number">1.0</span>)</span><br><span class="line"><span class="comment"># 50</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一个区间，如果数字不在这个区间内，就替换为区间的最小/最大值</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.clip(a, a_min=<span class="number">10</span>, a_max=<span class="number">30</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.where(a &lt; <span class="number">10</span>, <span class="number">10</span>, np.where(a &gt; <span class="number">30</span>, <span class="number">30</span>, a))</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  30.        ,  30.        ,  17.46948556,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.26877027,  10.        ,  13.36889132,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.7152288 ,  10.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  29.26817347])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计前n大的元素位置</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">a.argsort()[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># array([ 2, 15, 18,  1, 16], dtype=int64)</span></span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.argpartition(-a, <span class="number">5</span>)[:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([16,  1, 18, 15,  2], dtype=int64)</span></span><br><span class="line"><span class="comment"># 获取前n大的元素</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法1:</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法2:</span></span><br><span class="line">np.sort(a)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法3:</span></span><br><span class="line">np.partition(a, kth=-<span class="number">5</span>)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 前三个输出都是</span></span><br><span class="line"><span class="comment"># array([ 41.06481956,  44.63016661,  47.06146116,  44.63675129,  49.06512199])</span></span><br><span class="line"><span class="comment"># 方法4:</span></span><br><span class="line">a[np.argpartition(-a, <span class="number">5</span>)][:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([ 49.06512199,  47.06146116,  44.63675129,  44.63016661,  41.06481956])</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算有唯一值的行数,没看懂</span></span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">11</span>,size=(<span class="number">6</span>, <span class="number">10</span>))</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[ 2,  6,  5,  3,  9,  4,  6,  1, 10,  4],</span></span><br><span class="line"><span class="comment">#        [ 7,  4,  5,  8,  7,  4, 10,  1,  5,  5],</span></span><br><span class="line"><span class="comment">#        [ 6,  8,  7,  7,  3,  5,  3,  8,  2,  7],</span></span><br><span class="line"><span class="comment">#        [ 7,  1,  8,  3,  4,  6,  5,  3,  5,  4],</span></span><br><span class="line"><span class="comment">#        [ 8, 10,  1,  1,  6, 10,  7,  7,  6,  7],</span></span><br><span class="line"><span class="comment">#        [ 5,  8,  4, 10,  3,  4,  9,  8,  2,  6]])</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">counts_of_all_values_rowwise</span>(<span class="params">arr2d</span>):</span><br><span class="line">    num_counts_array = [np.unique(row, return_counts=<span class="literal">True</span>) <span class="keyword">for</span> row <span class="keyword">in</span> arr2d]</span><br><span class="line">    <span class="keyword">return</span> [[<span class="built_in">int</span>(b[a==i]) <span class="keyword">if</span> i <span class="keyword">in</span> a <span class="keyword">else</span> <span class="number">0</span> <span class="keyword">for</span> i <span class="keyword">in</span> np.unique(arr2d)] <span class="keyword">for</span> a, b <span class="keyword">in</span> num_counts_array]</span><br><span class="line">counts_of_all_values_rowwise(arr)</span><br><span class="line"><span class="comment"># [[1, 1, 1, 2, 1, 2, 0, 0, 1, 1],</span></span><br><span class="line"><span class="comment">#  [1, 0, 0, 2, 3, 0, 2, 1, 0, 1],</span></span><br><span class="line"><span class="comment">#  [0, 1, 2, 0, 1, 1, 3, 2, 0, 0],</span></span><br><span class="line"><span class="comment">#  [1, 0, 2, 2, 2, 1, 1, 1, 0, 0],</span></span><br><span class="line"><span class="comment">#  [2, 0, 0, 0, 0, 2, 3, 1, 0, 2],</span></span><br><span class="line"><span class="comment">#  [0, 1, 1, 2, 1, 1, 0, 2, 1, 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将多维数组转换为一维数组</span></span><br><span class="line">arr1 = np.arange(<span class="number">3</span>)</span><br><span class="line">arr2 = np.arange(<span class="number">3</span>,<span class="number">7</span>)</span><br><span class="line">arr3 = np.arange(<span class="number">7</span>,<span class="number">10</span>)</span><br><span class="line">array_of_arrays = np.array([arr1, arr2, arr3])</span><br><span class="line">array_of_arrays      <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([array([0, 1, 2]), array([3, 4, 5, 6]), array([7, 8, 9])], dtype=object)</span></span><br><span class="line">arr_2d = np.concatenate(array_of_arrays)</span><br><span class="line">arr_2d</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-5">51-60</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 唯一编码位置</span></span><br><span class="line">np.random.seed(<span class="number">101</span>)</span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">4</span>, size=<span class="number">6</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([2, 3, 2, 2, 2, 1])</span></span><br><span class="line"><span class="comment"># 返回数组是3x6,因为数组长度为6，最大数为1。1对应的是[1,0,0,],2对应的是[0,1,0],3对应的是[0,0,1]</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">one_hot_encodings</span>(<span class="params">arr</span>):</span><br><span class="line">    uniqs = np.unique(arr)</span><br><span class="line">    out = np.zeros((arr.shape[<span class="number">0</span>], uniqs.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">for</span> i, k <span class="keyword">in</span> <span class="built_in">enumerate</span>(arr):</span><br><span class="line">        out[i, k-<span class="number">1</span>] = <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> out</span><br><span class="line">one_hot_encodings(arr)</span><br><span class="line"><span class="comment"># array([[ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  0.,  1.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 1.,  0.,  0.]])</span></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">(arr[:, <span class="literal">None</span>] == np.unique(arr)).view(np.int8)</span><br><span class="line"><span class="comment"># array([[0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 0, 1],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [1, 0, 0]], dtype=int8)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取由变量名分组的行号</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">species = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;str&#x27;</span>, usecols=<span class="number">4</span>)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"></span><br><span class="line">[i <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> i, grp <span class="keyword">in</span> <span class="built_in">enumerate</span>(species_small[species_small==val])]</span><br><span class="line">[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据已知分类组名创建索引</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"><span class="comment"># 方法一，用列表解析式</span></span><br><span class="line">output = [np.argwhere(np.unique(species_small) == s).tolist()[<span class="number">0</span>][<span class="number">0</span>] <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]]</span><br><span class="line"><span class="comment"># 方法二，不用列表解析式直接循环</span></span><br><span class="line">output = []</span><br><span class="line">uniqs = np.unique(species_small)</span><br><span class="line"><span class="keyword">for</span> val <span class="keyword">in</span> uniqs:  <span class="comment"># uniq values in group</span></span><br><span class="line">    <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]:  <span class="comment"># each element in group</span></span><br><span class="line">        groupid = np.argwhere(uniqs == s).tolist()[<span class="number">0</span>][<span class="number">0</span>]  <span class="comment"># groupid</span></span><br><span class="line">        output.append(groupid)</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数值排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=<span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [ 9  4 15  0 17 16 17  8  9  0]</span></span><br><span class="line">a.argsort().argsort()</span><br><span class="line"><span class="comment"># [4 2 6 0 8 7 9 3 5 1]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 多维数组排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=[<span class="number">2</span>,<span class="number">5</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line">a.ravel().argsort().argsort().reshape(a.shape)</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line"><span class="comment"># [[4 2 6 0 8]</span></span><br><span class="line"><span class="comment">#  [7 9 3 5 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一行的最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">np.amax(a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.apply_along_axis(np.<span class="built_in">max</span>, arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([9, 8, 6, 3, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每行最小值比最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line">np.apply_along_axis(<span class="keyword">lambda</span> x: np.<span class="built_in">min</span>(x)/np.<span class="built_in">max</span>(x), arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([ 0.44444444,  0.125     ,  0.5       ,  1.        ,  0.11111111])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中重复的数值，输出布尔型</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">0</span>, <span class="number">5</span>, <span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [0 0 3 0 2 4 2 2 2 2]</span></span><br><span class="line">out = np.full(a.shape[<span class="number">0</span>], <span class="literal">True</span>)</span><br><span class="line">unique_positions = np.unique(a, return_index=<span class="literal">True</span>)[<span class="number">1</span>]</span><br><span class="line">out[unique_positions] = <span class="literal">False</span></span><br><span class="line">out</span><br><span class="line"><span class="comment"># [False  True False  True False False  True  True  True  True]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一分类的平均值</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">numeric_column = iris[:, <span class="number">1</span>].astype(<span class="string">&#x27;float&#x27;</span>)  <span class="comment"># sepalwidth</span></span><br><span class="line">grouping_column = iris[:, <span class="number">4</span>]  <span class="comment"># species</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">output = [[group_val, numeric_column[grouping_column==group_val].mean()] <span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column)]</span><br><span class="line"><span class="comment"># 方法二：不用列表解析直接循环</span></span><br><span class="line">output = []</span><br><span class="line"><span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column):</span><br><span class="line">    output.append([group_val, numeric_column[grouping_column==group_val].mean()])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [[b&#x27;Iris-setosa&#x27;, 3.418],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-versicolor&#x27;, 2.770],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-virginica&#x27;, 2.974]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将图像转换为数组存储，并再转换出图像显示</span></span><br><span class="line"><span class="keyword">from</span> io <span class="keyword">import</span> BytesIO</span><br><span class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="keyword">import</span> PIL, requests</span><br><span class="line"></span><br><span class="line">URL = <span class="string">&#x27;https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg&#x27;</span></span><br><span class="line">response = requests.get(URL)</span><br><span class="line">I = Image.<span class="built_in">open</span>(BytesIO(response.content))</span><br><span class="line">I = I.resize([<span class="number">150</span>,<span class="number">150</span>])</span><br><span class="line">arr = np.asarray(I)</span><br><span class="line">im = PIL.Image.fromarray(np.uint8(arr))</span><br><span class="line">Image.Image.show(im)</span><br></pre></td></tr></table></figure>
-<h2 id="section-6">61-70</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 去除数组中所有的缺失值</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,np.nan,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,np.nan])</span><br><span class="line">a[~np.isnan(a)]</span><br><span class="line">array([ <span class="number">1.</span>,  <span class="number">2.</span>,  <span class="number">3.</span>,  <span class="number">5.</span>,  <span class="number">6.</span>,  <span class="number">7.</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算两个数组间的欧几里得</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>,<span class="number">5</span>])</span><br><span class="line">b = np.array([<span class="number">4</span>,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,<span class="number">8</span>])</span><br><span class="line">dist = np.linalg.norm(a-b)</span><br><span class="line">dist</span><br><span class="line"><span class="comment"># 6.7082039324993694</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中最大的两个数</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">3</span>, <span class="number">7</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">6</span>, <span class="number">0</span>, <span class="number">1</span>])</span><br><span class="line">doublediff = np.diff(np.sign(np.diff(a)))</span><br><span class="line">peak_locations = np.where(doublediff == -<span class="number">2</span>)[<span class="number">0</span>] + <span class="number">1</span></span><br><span class="line">peak_locations</span><br><span class="line"><span class="comment"># array([2, 5])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组数组数值每项减去一维数组对应的行号</span></span><br><span class="line">a_2d = np.array([[<span class="number">3</span>,<span class="number">3</span>,<span class="number">3</span>],[<span class="number">4</span>,<span class="number">4</span>,<span class="number">4</span>],[<span class="number">5</span>,<span class="number">5</span>,<span class="number">5</span>]])</span><br><span class="line">b_1d = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">a_2d - b_1d[:,<span class="literal">None</span>]</span><br><span class="line"><span class="comment"># [[2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组中重复的第n次的数的索引</span></span><br><span class="line">x = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>])</span><br><span class="line">n = <span class="number">5</span></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">[i <span class="keyword">for</span> i, v <span class="keyword">in</span> <span class="built_in">enumerate</span>(x) <span class="keyword">if</span> v == <span class="number">1</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 方法二：numpy方法</span></span><br><span class="line">np.where(x == <span class="number">1</span>)[<span class="number">0</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 8</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将`datetime64`格式转换为`datetime`</span></span><br><span class="line">dt64 = np.datetime64(<span class="string">&#x27;2018-02-25 22:10:10&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一：利用datetime模块</span></span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line">dt64.tolist()</span><br><span class="line"><span class="comment"># 方法二：numpy直接转换</span></span><br><span class="line">dt64.astype(datetime)</span><br><span class="line">datetime.datetime(<span class="number">2018</span>, <span class="number">2</span>, <span class="number">25</span>, <span class="number">22</span>, <span class="number">10</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这个没看懂。。</span></span><br><span class="line"><span class="comment"># 如何计算numpy数组的移动平均值(累加和)</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">Z = np.random.randint(<span class="number">10</span>, size=<span class="number">10</span>)</span><br><span class="line">Z</span><br><span class="line"><span class="comment"># [8 8 3 7 7 0 4 2 5 2]</span></span><br><span class="line"><span class="comment"># 方法一: np.cumsum</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">moving_average</span>(<span class="params">a, n=<span class="number">3</span></span>) :</span><br><span class="line">    ret = np.cumsum(a, dtype=<span class="built_in">float</span>)</span><br><span class="line">    ret[n:] = ret[n:] - ret[:-n]</span><br><span class="line">    <span class="keyword">return</span> ret[n - <span class="number">1</span>:] / n</span><br><span class="line">moving_average(Z, n=<span class="number">3</span>).<span class="built_in">round</span>(<span class="number">2</span>)</span><br><span class="line"><span class="comment"># 方法二: np.convolve</span></span><br><span class="line">np.convolve(Z, np.ones(<span class="number">3</span>)/<span class="number">3</span>, mode=<span class="string">&#x27;valid&#x27;</span>)</span><br><span class="line"><span class="comment"># [ 6.33  6.    5.67  4.67  3.67  2.    3.67  3.  ]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个数组，只给出起始值、步长、数组长度</span></span><br><span class="line">length = <span class="number">10</span></span><br><span class="line">start = <span class="number">5</span></span><br><span class="line">step = <span class="number">3</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seq</span>(<span class="params">start, length, step</span>):</span><br><span class="line">    end = start + (step*length)</span><br><span class="line">    <span class="keyword">return</span> np.arange(start, end, step)</span><br><span class="line">seq(start, length, step)</span><br><span class="line"><span class="comment"># array([ 5,  8, 11, 14, 17, 20, 23, 26, 29, 32])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy数组填补不完整的日期</span></span><br><span class="line">dates = np.arange(np.datetime64(<span class="string">&#x27;2018-02-01&#x27;</span>), np.datetime64(<span class="string">&#x27;2018-02-25&#x27;</span>), <span class="number">2</span>)</span><br><span class="line">dates</span><br><span class="line"><span class="comment"># [&#x27;2018-02-01&#x27; &#x27;2018-02-03&#x27; &#x27;2018-02-05&#x27; &#x27;2018-02-07&#x27; &#x27;2018-02-09&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-11&#x27; &#x27;2018-02-13&#x27; &#x27;2018-02-15&#x27; &#x27;2018-02-17&#x27; &#x27;2018-02-19&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-21&#x27; &#x27;2018-02-23&#x27;]</span></span><br><span class="line"><span class="comment"># 方法一: 列表解析</span></span><br><span class="line">filled_in = np.array([np.arange(date, (date+d)) <span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates))]).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line"><span class="comment"># 方法二: 不用列表解析循环</span></span><br><span class="line">out = []</span><br><span class="line"><span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates)):</span><br><span class="line">    out.append(np.arange(date, (date+d)))</span><br><span class="line">filled_in = np.array(out).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># array([&#x27;2018-02-01&#x27;, &#x27;2018-02-02&#x27;, &#x27;2018-02-03&#x27;, &#x27;2018-02-04&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-05&#x27;, &#x27;2018-02-06&#x27;, &#x27;2018-02-07&#x27;, &#x27;2018-02-08&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-09&#x27;, &#x27;2018-02-10&#x27;, &#x27;2018-02-11&#x27;, &#x27;2018-02-12&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-13&#x27;, &#x27;2018-02-14&#x27;, &#x27;2018-02-15&#x27;, &#x27;2018-02-16&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-17&#x27;, &#x27;2018-02-18&#x27;, &#x27;2018-02-19&#x27;, &#x27;2018-02-20&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-21&#x27;, &#x27;2018-02-22&#x27;, &#x27;2018-02-23&#x27;], dtype=&#x27;datetime64[D]&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一维数组，变形为二维数组（给定每维长度）</span></span><br><span class="line">arr = np.arange(<span class="number">15</span>)</span><br><span class="line">arr</span><br><span class="line">array([ <span class="number">0</span>,  <span class="number">1</span>,  <span class="number">2</span>,  <span class="number">3</span>,  <span class="number">4</span>,  <span class="number">5</span>,  <span class="number">6</span>,  <span class="number">7</span>,  <span class="number">8</span>,  <span class="number">9</span>, <span class="number">10</span>, <span class="number">11</span>, <span class="number">12</span>, <span class="number">13</span>, <span class="number">14</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gen_strides</span>(<span class="params">arr, stride_len=<span class="number">5</span>, window_len=<span class="number">5</span></span>):</span><br><span class="line">    n_strides = (a.size-window_len)//stride_len + <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> np.array([a[s:(s+window_len)] <span class="keyword">for</span> s <span class="keyword">in</span> np.arange(<span class="number">0</span>, n_strides*stride_len, stride_len)])</span><br><span class="line">gen_strides(np.arange(<span class="number">15</span>), stride_len=<span class="number">2</span>, window_len=<span class="number">4</span>)</span><br><span class="line"><span class="comment"># [[ 0  1  2  3]</span></span><br><span class="line"><span class="comment">#  [ 2  3  4  5]</span></span><br><span class="line"><span class="comment">#  [ 4  5  6  7]</span></span><br><span class="line"><span class="comment">#  [ 6  7  8  9]</span></span><br><span class="line"><span class="comment">#  [ 8  9 10 11]</span></span><br><span class="line"><span class="comment">#  [10 11 12 13]]</span></span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>Numpy</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>优化pandas内存入门</title>
-    <url>/hexo_blog/2018/03/17/PythonPackages/%E4%BC%98%E5%8C%96pandas%E5%86%85%E5%AD%98%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>降低数据存储消耗并提高处理效率，本文教你如何通过优化数值类型、<code>object</code>类型和
-<code>datetime</code>类型三个方面来减少内存使用量。其中，使用
-<code>Categoricals</code>优化
-<code>object</code>类型数据是效果最显著的一种优化方式。另外，还介绍了如何在读取数据的时候就完成优化步骤，避免加载完数据再做优化的麻烦。</p>
-<span id="more"></span>
-<h2 id="简单的概念">简单的概念</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"></span><br><span class="line"><span class="comment"># 读取数据</span></span><br><span class="line">df = pd.read_csv(<span class="string">&quot;test.csv&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 得到精确的内存信息</span></span><br><span class="line">df.info(memory_usage=<span class="string">&#x27;deep&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 说明：之后都用df来表示读取到的dataframe</span></span><br></pre></td></tr></table></figure>
-<p>pandas中每一个数据类型都有一个专门的类来处理。</p>
-<ul>
-<li><code>ObjectBlock</code>： 字符串列的块</li>
-<li><code>FloatBlock</code>： 浮点数列的块</li>
-<li><code>Numpy ndarray</code>：整型和浮点数值的块（非常快，用C数组构建的）</li>
-</ul>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> dtype <span class="keyword">in</span> [<span class="string">&#x27;float&#x27;</span>, <span class="string">&#x27;int&#x27;</span>, <span class="string">&#x27;object&#x27;</span>]:</span><br><span class="line">    <span class="comment"># 选中对应的dtype列</span></span><br><span class="line">    selected_dtype = df.select_dtypes(include=[dtype])</span><br><span class="line">    <span class="comment"># 查看内存使用量的平均值</span></span><br><span class="line">    mean_usage_b = selected_dtype.memory_usage(deep=<span class="literal">True</span>).mean()</span><br><span class="line">    <span class="comment"># 获取到的数据单位为K, 这里转换一下</span></span><br><span class="line">    mean_usage_mb = mean_usage_b / <span class="number">1024</span> ** <span class="number">2</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;Average memory usage for &#123;&#125; columns:&#123;:03.2f&#125; MB&quot;</span>.<span class="built_in">format</span>(dtype, mean_usage_mb))</span><br></pre></td></tr></table></figure>
-<p>这里我们可以发现 <code>object</code>类占用内存最大</p>
-<h3 id="子类型subtype">子类型(subtype)</h3>
-<table>
-<thead>
-<tr>
-<th>memory usage</th>
-<th>float</th>
-<th>int</th>
-<th>uint</th>
-<th>datetime</th>
-<th>bool</th>
-<th>object</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>1 bytes</td>
-<td></td>
-<td>int8</td>
-<td>uint8</td>
-<td></td>
-<td>bool</td>
-<td></td>
-</tr>
-<tr>
-<td>2 bytes</td>
-<td>float16</td>
-<td>int16</td>
-<td>uint16</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>4 bytes</td>
-<td>float32</td>
-<td>int32</td>
-<td>uint32</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>8 bytes</td>
-<td>float64</td>
-<td>int64</td>
-<td>uint64</td>
-<td>datetime64</td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>variable</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td>object</td>
-</tr>
-</tbody>
-</table>
-<p>一个
-<code>int8</code>类型值使用1个字节的存储空间，可以表示256(2^8)个二进制数，即-128到127的所有整数值。根据表可以得出，对于每个列，应该尽可能"抠"一点，
-物尽其用。这里的
-<code>uint</code>表示无符号整型，可以更有效地处理正整数的列。</p>
-<h2 id="优化">优化</h2>
-<h3 id="数值型数据">数值型数据</h3>
-<p>这里可以使用 <code>to_numeric()</code>对数值类型进行
-<code>downcast</code>（向下转型）的操作。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中整数型的列</span></span><br><span class="line">df_int_columns = df.select_dtypes(include=[<span class="string">&#x27;int&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_int_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;unsigned&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中浮点型数据</span></span><br><span class="line">df_float_columns = df.select_dtypes(include=[<span class="string">&#x27;float&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_float_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;float&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<p>这里能够有效的利用内存，但是缺点很明显的就是只能用于数值型数据，而且优化的空间有限。</p>
-<h3 id="object型数据"><code>object</code>型数据</h3>
-<p><code>object</code>类型实际上使用的是Python字符串对应的值，由于Python解释性语言的特性，字符串存储方式很碎片化（消耗更多内存，访问速度更慢）。<code>object</code>列中的每个元素都是一个指针。<code>object</code>类型的数据内存使用情况是可变的。如果每个字符串是单独存储的，那么实际上字符串占用内存是很大。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">In [<span class="number">1</span>]: <span class="keyword">from</span> sys <span class="keyword">import</span> getsizeof</span><br><span class="line"><span class="comment"># 字符串本身内存使用情况</span></span><br><span class="line">In [<span class="number">2</span>]: s1 = <span class="string">&quot;hello world!&quot;</span></span><br><span class="line">In [<span class="number">3</span>]: s2 = <span class="string">&quot;hello pandas!&quot;</span></span><br><span class="line">In [<span class="number">4</span>]: getsizeof(s1), getsizeof(s2)</span><br><span class="line">Out[<span class="number">4</span>]: (<span class="number">61</span>, <span class="number">62</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串(object)在pandas中内存使用情况</span></span><br><span class="line">In [<span class="number">9</span>]: <span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line">In [<span class="number">10</span>]: ser = pd.Series([s1, s2])</span><br><span class="line">In [<span class="number">11</span>]: ser.apply(getsizeof)</span><br><span class="line">Out[<span class="number">11</span>]:</span><br><span class="line"><span class="number">0</span>    <span class="number">61</span></span><br><span class="line"><span class="number">1</span>    <span class="number">62</span></span><br><span class="line">dtype: int64</span><br></pre></td></tr></table></figure>
-<p>这里通过观察看到，在这里pandas使用了int64来存储，实际占用大小与字符串本身是一样的。这里可以使用
-<code>Categoricals</code>来优化
-<code>object</code>类型。<code>Categoricals</code>的工作原理我理解为，某个
-<code>object</code>类有有限的分类情况(比如只有
-<code>red</code>、<code>yellow</code>、<code>blue</code>、<code>black</code>等颜色相关)，那么
-<code>Categoricals</code>将这些分类 <code>object</code>对象转换为
-<code>int</code>子类型（对应上面的0, 1, 2, 3）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中object类型</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="comment"># 查看object类型列的相关信息</span></span><br><span class="line">df_obj_columns.describe()</span><br></pre></td></tr></table></figure>
-<p>如果
-<code>unique</code>(不同值)的数量很少，那么就可以使用这种优化方案。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设某一列符合优化条件，为df_obj_less，使类型转换为category</span></span><br><span class="line">df_category_column = df_obj_less.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"><span class="comment"># 转换之后，主观观察数据没有什么区别，只是类型改变了</span></span><br><span class="line"><span class="comment"># 观察转换之后实际上的数据</span></span><br><span class="line">df_category_column.cat.codes</span><br><span class="line"><span class="comment"># 变成了由0,1,2等数字构成的int8类型的数据</span></span><br></pre></td></tr></table></figure>
-<p>这里提升的空间远远超过第一步优化的空间，具体要结合数据来检验（最好貌似可以减少98%的使用量）。这里有个很大的缺点就是无法进行数值计算，即没有办法使用
-<code>pd.Series.min()</code>、<code>pd.Series.max()</code>等与数值相关操作。</p>
-<p>还有一个问题就是在有多少个
-<code>unique</code>(不同值)的情况下才使用这种方法。首先，毫无疑问的是如果需要计算操作的列是不能使用的。第二是如果
-<code>unique</code>的比例小于50%(个人觉得比例应该更小)就可以使用这种情况，如果过多的话，转换之后消耗的内存会更多（不仅需要存储string还有int）。</p>
-<p>附上筛选 <code>unique</code>比例小于50%的代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取unique少于50%的object列</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="keyword">for</span> obj_col <span class="keyword">in</span> df_obj_columns:</span><br><span class="line">    unique_values = <span class="built_in">len</span>(df_obj_columns[obj_col].unique())</span><br><span class="line">    total_values = <span class="built_in">len</span>(df_obj_columns[obj_col])</span><br><span class="line">    <span class="keyword">if</span> unique_values / total_values &lt; <span class="number">0.5</span>:</span><br><span class="line">        obj_col.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建议提前copy一份数据，再做上面的操作</span></span><br></pre></td></tr></table></figure>
-<h3 id="datetime类型"><code>datetime</code>类型</h3>
-<p>这里其实不能叫做优化，因为这里是将数值型数据转换为
-<code>datetime</code>类型，虽然提高了内存的使用，但是转换为
-<code>datetime</code>类型的数据更容易进行分析（时间序列分析）。当然如果有
-<code>datetime</code>类型不在分析的范围内，自然可以无视。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设df_num_col为需要转换的列, 这里format格式看情况更改</span></span><br><span class="line">df_num_col = pd.to_datetime(df_num_col, <span class="built_in">format</span>=<span class="string">&quot;%Y%m%d&quot;</span>)</span><br></pre></td></tr></table></figure>
-<h2 id="总结">总结</h2>
-<p>写了这么多，如果每次等加载完数据再做优化操作，感觉有些鸡肋。但是其实可以使用
-<code>read_csv</code>等读取函数的几个参数来帮助我们在读取的时候就完成优化步骤。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先，需要整理出每一列最终的数据类型，组成一个dict</span></span><br><span class="line">dtypes_lst = [col.name <span class="keyword">for</span> name <span class="keyword">in</span> df.dtypes]</span><br><span class="line">column_types = <span class="built_in">dict</span>(<span class="built_in">zip</span>(dtypes.index, dtypes_lst))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重新读取数据, dtype指定数据格式，parse_dates指定列转换格式为datetime,infer_datetime_format尝试解析字符串形式的datatime格式（在一些情况下可以加快解析速度5-10倍）</span></span><br><span class="line">new_df = pd.read_csv(<span class="string">&#x27;test.csv&#x27;</span>, dtpye=column_types, parse_dates=[<span class="string">&#x27;date&#x27;</span>], infer_datetime_format=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>pandas</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Leetcode 字符串刷题记录</title>
-    <url>/hexo_blog/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</url>
-    <content><![CDATA[<p>字符串刷题分类</p>
-<span id="more"></span>
-<h2 id="滑动窗口">滑动窗口</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-substring-without-repeating-characters/">3.
-无重复字符的最长子串</a></p>
-<p>从左向右滑，需要一个数据结构（A）记录已经遍历的字符，还有一个变量（B）记录最大长度</p>
-<p>注：每次不需要从头开始，而是可以从重复那个字符的后一个开始，所以A需要记录一下索引</p>
-<h2 id="kmp">KMP</h2>
-<h2 id="动态规划">动态规划</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-palindromic-substring/">5.
-最长回文子串</a></p>
-<p>两个指针从两头开始，所以需要二维DP，元素是bool判断</p>
-<p><a href="https://leetcode.cn/problems/interleaving-string/">97.
-交错字符串</a></p>
-<p>两个字符串组成新的字符串，二维DP，元素是bool判断</p>
-<p><a href="https://leetcode.cn/problems/word-break/">139.
-单词拆分</a></p>
-<p>单词数组，一个一个单词拼接，组成最后单词，一维DP，元素是bool判断</p>
-<p><a
-href="https://leetcode.cn/problems/longest-palindromic-subsequence/">516.
-最长回文子序列</a></p>
-<p>回文，需要两个指针，二维DP，从i到j组成回文的长度，元素是长度，最小长度为1，相等+2</p>
-<p><a href="https://leetcode.cn/problems/ones-and-zeroes/">474.
-一和零</a></p>
-<p>字符串数组，组成数量限制（两个限制），所以限制需要二个维度，数组作为最外层那个维度，元素是个数，最小为0</p>
-<p><a
-href="https://leetcode.cn/problems/delete-operation-for-two-strings/">583.
-两个字符串的删除操作</a></p>
-<p>两个字符串，两个维度，删除次数，初始情况，删除个数为字符串长度。</p>
-<p><a href="https://leetcode.cn/problems/palindromic-substrings/">647.
-回文子串</a></p>
-<p>回文，二维DP，元素是bool判断，统计True个数</p>
-<p><a
-href="https://leetcode.cn/problems/longest-common-subsequence/">1143.
-最长公共子序列</a></p>
-<p>两个字符串，二维DP，长度</p>
-<p><a href="https://leetcode.cn/problems/jump-game-vii/">1871. 跳跃游戏
-VII</a></p>
-<p>能否到达终点，一维DP，反向思考，能否到达该点，所以元素为bool。+前缀和优化时间</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 回文串</span></span><br><span class="line">dp = [[<span class="literal">False</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">    dp[i][i] = <span class="literal">True</span></span><br><span class="line">    <span class="comment"># 从i到j是否组成回文串</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n-<span class="number">1</span>)[::-<span class="number">1</span>]:</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(i+<span class="number">1</span>, n):</span><br><span class="line">            <span class="keyword">if</span> s[i] == s[j] <span class="keyword">and</span> (j - i &lt;= <span class="number">2</span> <span class="keyword">or</span> dp[i+<span class="number">1</span>][j-<span class="number">1</span>]):</span><br><span class="line">                dp[i][j] = <span class="literal">True</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h2 id="双指针">双指针</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-repeating-character-replacement/">424.
-替换后的最长重复字符</a></p>
-<p>枚举右端点，如果没超过限制就继续移动右指针，否则移动左指针</p>
-<h2 id="栈">栈</h2>
-<p>PASS</p>
-<h2 id="字典树">字典树</h2>
-<p>PASS</p>
-<h2 id="模拟深搜">模拟/深搜</h2>
-<p>PASS</p>
-]]></content>
-      <categories>
-        <category>Algorithm</category>
-      </categories>
-      <tags>
-        <tag>字符串</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>海量数据处理问题（HashMap+分治法、位图法）</title>
-    <url>/hexo_blog/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</url>
-    <content><![CDATA[<p>从海量数据中找出不重复的数和出现次数最多的数的解决方案，分别采用了分治法+HashMap和位图法Bitmap两种方法。其中，位图法在大数据量下处理速度更快，内存占用更小。而对于其他类似问题，可以尝试使用分治法加最小堆/最大堆的方式处理。</p>
-<span id="more"></span>
-<p>原文链接：<a
-href="https://juejin.im/post/5aa0ee9f518825557c010bc0">经典面试问题: Top
-K 之 ---- 海量数据找出现次数最多或，不重复的。</a></p>
-<p>在数据处理中，较为棘手的有这么一种问题，海量数据如何进行处理。一旦数据的数量足够大，我们就不能使用常规的思路来解决。典型例子就是排序算法，在数据不大（不超过内存）的情况下，可以直接使用排序算法，但是当数据量很大时，我们就需要使用如多路归并此类的算法。</p>
-<p>总结问题：</p>
-<ol type="1">
-<li>内存一次性不能放下所有的数</li>
-<li>机器CPU核数不够</li>
-<li>……</li>
-</ol>
-<h2 id="不重复数">不重复数</h2>
-<p>在2.5亿个正整数中找出不重复的数。</p>
-<h3 id="分治法hashmap">分治法+HashMap</h3>
-<h4 id="说明">说明</h4>
-<p>比如将这些数据平均分为100个批次，每个批次就有250万数。每个批次用循环进行遍历，存入
-<code>HashMap&lt;a, b&gt;</code>，<code>a</code>表示每个数，<code>b</code>表示对应数出现次数，默认为1。每遍历完一批，就对当前
-<code>HashMap</code>进行去重，如果
-<code>b &gt; 1</code>，就排除此数。全部批次进行完筛选之后，剩下的数最后可以直接进行统计，得出不重复的数。</p>
-<h4 id="双间复杂度">双间复杂度</h4>
-<p>时间复杂度：250W * 100轮 + 其他批次。
-这里可以使用多核机器加速运算。</p>
-<p>空间复杂度：int保存数，<code>Key + Value（250W * 4Byte）/ (1024 * 1024)</code>约为
-<code>(Key + 9.5M)</code>内存</p>
-<h3 id="位图法-bitmap">位图法 Bitmap</h3>
-<h4 id="说明-1">说明</h4>
-<p>设计每两个
-<code>bit</code>位，标识一个数的出现情况。如：00表示没有出现，01表示出现一次，10表述出现多次。此题给出的是正整数，所以就是无符号的整型数，占四个字节。</p>
-<p>内存的计算。1B=8b, 4B=32b, 它可以表示的最大整数是<span
-class="math inline">\(2^{32}-1\)</span>, 即需要$2^{32}-1 $约位<span
-class="math inline">\(2^{32}\)</span>个<strong>位</strong>来表示这2.5亿个数，每个状态是两个数，总共就是<span
-class="math inline">\(2^{32}\times 2\)</span>个位。</p>
-<p>一次性申请的位图内存是: $2^{32} $ bit, 即<span
-class="math inline">\(\frac{2^{32}\times2}{(1024\times1024 \times
-8)}=1\)</span> GB。如果加上分治法，甚至都不需要1G内存。</p>
-<p>假设一个数是64, 则对应的bit位是 <span class="math inline">\(64\times
-2=128\)</span>，即127和128共同标识这个数的出现状态（没有出现、出现一次、出现多次）。所以，每当读入了一个数，先找对应的bit位，如果是00就改为01，如果是01就改为10，如果是10就不需要修改。最后扫描整个位图，如果是10，就将下标/2得出出现次数为1的这个数。</p>
-<h2 id="出现次数最多的">出现次数最多的</h2>
-<ol type="1">
-<li>找出一篇文章中，出现次数最多的单词</li>
-<li>10亿个正整数找出重复次数最多的100个整数</li>
-</ol>
-<p>这类问题被叫做Top K问题</p>
-<h3 id="分治法hashmap-1">分治法+HashMap</h3>
-<p>对于文章来说，我们需要规定每批次的字数限制。这里假设100为一批的处理极限，每次处理100个，如果最后一批少于100个就直接当作100来处理。之后每批文字进行切分(<code>split</code>)，比如英文的切割方法可以是空格、标点符号什么的，中文的切割方式可以是空格或者有预先准备好的词库。</p>
-<p>之后就跟上面不重复数字的思路类似了，遍历每批数据，存入
-<code>HashMap&lt;String, Integer&gt;</code>，String表示字符串，Integer对应出现的次数，最大的Integer就是出现次数最多的。</p>
-<p>对数字来说，就是跟不重复数字思路类似，将十亿个数分乘很多份，利用
-<code>HashMap&lt;int, int&gt;</code>来统计，最后找到第二个int最大的那个。每批筛选完之后，就进行排序，选出十亿个数中最大的。</p>
-<h3 id="位图法-bitmap-1">位图法 Bitmap</h3>
-<p>对于文章（字符串）来说，不建议采用</p>
-<p>对于数字来说，此题的背景不需要乘2，将每个数存进位图。出现了一次，bit位=1，没有出现就是0。但是没有办法统计出现次数，所以此题也不适合采用位图法。</p>
-<h2 id="其他">其他</h2>
-<p>在XXX中找出最大、最小、最大的几个、最小的几个。都可以使用分治法+最小堆/最大堆。</p>
-]]></content>
-      <categories>
-        <category>Algorithm</category>
-      </categories>
-      <tags>
-        <tag>HashMap</tag>
-        <tag>位图法</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Birthday in Pi：查找日期在Pi中的位置</title>
-    <url>/hexo_blog/2021/03/18/Python/Birthday_in_Pi/</url>
-    <content><![CDATA[<p>想知道自己的生日在圆周率π的第几位出现吗？本文介绍了从思路到实现所遇到的问题，详细分析了如何使用迭代器优化性能，并改用Python语言实现了类似“
-锁”内存等操作。最终，将函数打包成了exe程序，在公众号内提供查询服务，还提供了一个不断更新的生日与位置的查表。</p>
-<span id="more"></span>
-<p>完整的项目代码均开源: <a
-href="https://github.com/wnma3mz/birthday_in_pi">https://github.com/wnma3mz/birthday_in_pi</a></p>
-<h2 id="想法">想法</h2>
-<p>于我而言，很多项目的出发点都是源于浏览东西时所想，觉得有意思就去实现。比如这次是在即刻上看到北大数院公众号，可以回复生日（如20000101）就可以知道这串数字在Pi的第多少位出现。更详细的例子：已知3.1415926...，那么1415就是出现在第1位（从1开始计数，小数点后开始计数）。但该公众号仅在Pi的2亿位数中进行查找，所以有即友回复，搜不到结果。故想到可以试试更长的数据集是否能够搜到。</p>
-<p><strong>背景知识</strong></p>
-<p>理论上，Pi里面可以查找所有人的电话号码、生日、银行卡密码等等。</p>
-<h2 id="行动">行动</h2>
-<h3 id="数据">数据</h3>
-<p>巧妇难为无米之炊，这里最重要的就是Pi这个无理数数据。</p>
-<ol type="1">
-<li>是否有现成的数据文本-&gt;没搜到，搜到的位数都很小，不足以满足需求</li>
-<li>计算Pi（生成超过2亿数的文本）
-<ol type="1">
-<li>自己手写</li>
-<li>找程序。对于计算Pi的项目，本身就是一个科学问题，故一定有现有程序可以辅助完成。所以找到了<a
-href="http://www.numberworld.org/y-cruncher/">y-cruncher</a>项目。具体是否为当前最快的生成程序，未深究，不过实际测试，效率已经可以满足需求。（用i5-1035G1的CPU，生成25亿数，8个线程计算用了10分钟不到，文本大小2.4G）</li>
-</ol></li>
-</ol>
-<h3 id="编码">编码</h3>
-<p>该问题本身简单来看就是一个字符串查找问题，所以用Python的话，<code>find</code>函数可以直接完成这项任务。但一旦问题变"大"（数据集变大、机器变多......），就是一个"难"题了。</p>
-<p>当然，先不管如何，试试最粗暴的解法看看性能如何。根据测试结果，有两个问题，时间超过5s,
-内存占用2G（文本大小2G）。就算时间可以接受，那么这个空间占用也很难接受。试想，每查询一次都需要占用2G内存，查询完再释放。所以直接想到的优化点有以下几个</p>
-<ol type="1">
-<li>"锁"内存，即每次查询完不释放这个内存，这样IO时间就可以大大减小</li>
-<li>"做表"，查询的生日数字是有限的，无非就是(19700101-20101231有限数据)，就算100年，也只是36500个数据（实际会略高，因为有闰年）。查表的效率就非常快了</li>
-<li>从根本上优化这个程序</li>
-</ol>
-<h3 id="优化">优化</h3>
-<p>首先排除2，因为最终还是想查找任意数字，2不便于后续扩展，而且根据现有的性能，做表也需要很长时间。</p>
-<p>其次对于1，如果考虑对外开放，服务器占用2G内存空间，实在是太"奢侈"了</p>
-<p>所以最后就只能考虑3。</p>
-<h4 id="迭代器">迭代器</h4>
-<p>Python有个关键词是
-<code>yield</code>，它可以实现迭代器这个功能。说人话就是，它不是一次性把数据加载出来，而且每次只加载一部分。下面是一个简单的例子</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">yield_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">for_return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">return</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">return</span> lst</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    lst = <span class="built_in">list</span>(<span class="built_in">range</span>(<span class="number">10</span>))</span><br><span class="line">    <span class="built_in">print</span>(yield_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(for_return_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(return_func(lst))</span><br><span class="line">    <span class="keyword">for</span> x <span class="keyword">in</span> yield_func(lst):</span><br><span class="line">        <span class="built_in">print</span>(x, end=<span class="string">&quot; &quot;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line">&lt;generator object yield_func at 0x000001644AF624F8&gt;</span><br><span class="line">0</span><br><span class="line">[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span><br><span class="line">0 1 2 3 4 5 6 7 8 9</span><br></pre></td></tr></table></figure>
-<p>所以肯定是要利用这个方法来帮助优化程序。在Python操作文件的时候
-<code>readline()</code>函数就是一个生成器，一行一行的返回，查找速度比直接
-<code>read()</code>或者 <code>readlines()</code>更快。</p>
-<p>但这里的问题是，输出的Pi文本是一行。。。所以没法直接使用。当然也可以手动把这个文本按行切割后，再用
-<code>readline()</code>函数。</p>
-<p>这里我找到的解决方案[1]可以更方便的解决这个问题，详见博客or项目源码。</p>
-<h4 id="bug">bug</h4>
-<p>把程序写好之后，可以发现一个很明显的bug，就是查询的字符串如果在两个
-<code>chunk</code>之间，是会直接跳过。比如
-<code>12345678</code>，<code>block_size=4</code>，那么就是
-<code>1234</code>和 <code>5678</code>。如果查找
-<code>45</code>是找不到的。</p>
-<p>所以这里需要做两个额外的处理</p>
-<ol type="1">
-<li>加上上一个 <code>block_size</code>的字符串
-<code>old_chunk = copy.deepcopy(chunk[-len(target_str) :])</code>。这里为了省空间和时间，只加上了需要查询字符串的长度，就可以满足需求</li>
-<li>返回查找位数时需要额外
-<code>-len(old_chunk)</code>。（注：这里一开始忘了操作，故前几位查询的既有会发现结果与北大数院查询的结果差10位，在这里致歉）</li>
-</ol>
-<p>最后经过测试后，性能基本达到预期（见项目README，有说明具体结果）。</p>
-<h2 id="部署">部署</h2>
-<p>至此，核心程序已经完成，接下来需要考虑一种方式来给其他人方便使用了。最直接的其实就是写一个Web，前端去调用后端返回结果。但同样的，这里还是考虑到网站流量等原因，跟北大数院一样放到公众号上也很方便，开发上也只需要完成后端即可，无需考虑前端UI啥的。</p>
-<p>查询公众号文档后，一通操作，就把功能完成了（文案直接"抄"了北大数院的文案）。代码见<a
-href="https://github.com/wnma3mz/birthday_in_pi/blob/master/flask_server.py">flask_server.py</a></p>
-<p>这里需要在微信公众号后台配置开发者模式，然后把token复制进代码。</p>
-<h3 id="tip">Tip</h3>
-<p>公众号开发要求是80端口，但这里通过子域名的方式+nginx反代就可以直接用域名访问，而无需80端口。nginx配置见下</p>
-<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">listen</span>       <span class="number">80</span>;</span><br><span class="line">    <span class="attribute">server_name</span>  xxx.yyy.zzz;</span><br><span class="line"></span><br><span class="line">    <span class="section">location</span> / &#123;</span><br><span class="line">        <span class="attribute">uwsgi_pass</span> <span class="number">0.0.0.0</span>:port;</span><br><span class="line">        <span class="attribute">proxy_pass</span> http://0.0.0.0:port/wx_api;</span><br><span class="line">    &#125;</span><br><span class="line">  </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>这里的 <code>xxx.yyy.zzz</code>就是域名，port就是
-<code>flask</code>开启的端口</p>
-<h2 id="其他">其他</h2>
-<h3 id="查表">查表</h3>
-<p>当然，查表的方式还是最快的，所以还是建了一个表，并且做了一个简单的分析。分析内容如下</p>
-<p>最近只需246位能够找到,
-最远需要近9亿位。列出Top10（左侧为年月日，右侧为第x位）</p>
-<p>日期范围为19700101-20091231</p>
-<table>
-<thead>
-<tr>
-<th>日期</th>
-<th>位数</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>20190914</td>
-<td>246</td>
-</tr>
-<tr>
-<td>20190516</td>
-<td>5364</td>
-</tr>
-<tr>
-<td>19910403</td>
-<td>8365</td>
-</tr>
-<tr>
-<td>20141020</td>
-<td>9813</td>
-</tr>
-<tr>
-<td>19950728</td>
-<td>1,3874</td>
-</tr>
-<tr>
-<td>20121031</td>
-<td>1,4538</td>
-</tr>
-<tr>
-<td>19990814</td>
-<td>2,5593</td>
-</tr>
-<tr>
-<td>19810705</td>
-<td>2,7202</td>
-</tr>
-<tr>
-<td>19921122</td>
-<td>3,0884</td>
-</tr>
-<tr>
-<td>19930509</td>
-<td>3,6615</td>
-</tr>
-<tr>
-<td>..........</td>
-<td>..........</td>
-</tr>
-<tr>
-<td>19941227</td>
-<td>7,1993,2960</td>
-</tr>
-<tr>
-<td>20080923</td>
-<td>7,2930,2729</td>
-</tr>
-<tr>
-<td>19911201</td>
-<td>7,2985,0775</td>
-</tr>
-<tr>
-<td>20071206</td>
-<td>7,3782,1997</td>
-</tr>
-<tr>
-<td>20090220</td>
-<td>7,5402,4215</td>
-</tr>
-<tr>
-<td>19730829</td>
-<td>7,7357,0338</td>
-</tr>
-<tr>
-<td>20070526</td>
-<td>7,8655,8038</td>
-</tr>
-<tr>
-<td>19951122</td>
-<td>8,2582,0139</td>
-</tr>
-<tr>
-<td>20140407</td>
-<td>8,7320,8091</td>
-</tr>
-<tr>
-<td>19730521</td>
-<td>8,8950,7619</td>
-</tr>
-</tbody>
-</table>
-<p>为什么4位分隔，而不是3位？</p>
-<p>因为我觉得这样看的更舒服，读万、亿更方便。。。</p>
-<h3 id="程序打包">程序打包</h3>
-<p>为方便没有Python程序的用户，同时也不想使用公众号。在Windows上打包了一份exe程序。命令也很简单
-<code>pyinstaller -F .\read_large_pi.py</code>。</p>
-<p>最后的最后，该程序尚存在优化空间，当查询数字较长（超过8位）时，时间花费也更久（超过1s）。</p>
-<h2 id="参考文章">参考文章</h2>
-<p>[1] <a
-href="https://lovemma.github.io/posts/python-%E5%A6%82%E4%BD%95%E6%B5%81%E5%BC%8F%E8%AF%BB%E5%8F%96%E5%A4%A7%E6%96%87%E4%BB%B6/">python-如何流式读取大文件</a></p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>微信开发</tag>
-        <tag>Steaming</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python3的Tools-scripts整理</title>
-    <url>/hexo_blog/2020/06/14/Python/Python3%E7%9A%84Tools-scripts%E6%95%B4%E7%90%86/</url>
-    <content><![CDATA[<p>Python3的Tools-scripts整理</p>
-<span id="more"></span>
-<p>文件路径为<code>Anaconda3\Tools\scripts</code></p>
-<h3 id="byext.py">byext.py</h3>
-<p>分析文件/文件夹</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># py文件</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line">.py   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">TOTAL   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line"></span><br><span class="line"><span class="comment"># 二进制文件</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line">&lt;none&gt;      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line"> TOTAL      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件夹</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br><span class="line"> .png      <span class="number">2</span>   <span class="number">2823</span>             <span class="number">2</span></span><br><span class="line">  .py         <span class="number">10471</span>             <span class="number">5</span>    <span class="number">224</span>    <span class="number">647</span></span><br><span class="line"> .txt         <span class="number">17404</span>             <span class="number">3</span>    <span class="number">554</span>    <span class="number">960</span></span><br><span class="line">&lt;<span class="built_in">dir</span>&gt;                    <span class="number">1</span></span><br><span class="line">TOTAL      <span class="number">2</span>  <span class="number">30698</span>      <span class="number">1</span>     <span class="number">10</span>    <span class="number">778</span>   <span class="number">1607</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br></pre></td></tr></table></figure>
-<table>
-<thead>
-<tr>
-<th>输出</th>
-<th>主要语法</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>ext</td>
-<td><code>os.path.normcase(os.path.splitext[1])</code></td>
-</tr>
-<tr>
-<td>files</td>
-<td><code>os.path.isfile</code></td>
-</tr>
-<tr>
-<td>bytes</td>
-<td>以二进制(<code>rb</code>)读文件，<code>len</code>的输出</td>
-</tr>
-<tr>
-<td>lines</td>
-<td><code>str(data, 'latin-1').splitlines()</code></td>
-</tr>
-<tr>
-<td>words</td>
-<td><code>len(data.split())</code>的输出</td>
-</tr>
-<tr>
-<td>binary</td>
-<td><code>if *b*'\0' in data</code></td>
-</tr>
-<tr>
-<td>dirs</td>
-<td>os.path.isdir</td>
-</tr>
-<tr>
-<td>lnk</td>
-<td>os.path.islink</td>
-</tr>
-</tbody>
-</table>
-<h3 id="checkpip.py">checkpip.py</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">The latest version of setuptools on PyPI <span class="keyword">is</span> <span class="number">46.2</span><span class="number">.0</span>, but ensurepip has <span class="number">40.8</span><span class="number">.0</span></span><br><span class="line">The latest version of pip on PyPI <span class="keyword">is</span> <span class="number">20.1</span>, but ensurepip has <span class="number">19.0</span><span class="number">.3</span></span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 主要是检查版本</span></span><br><span class="line"><span class="keyword">import</span> ensurepip</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(ensurepip._PROJECTS)</span><br><span class="line"><span class="comment"># [(&#x27;setuptools&#x27;, &#x27;40.8.0&#x27;), (&#x27;pip&#x27;, &#x27;19.0.3&#x27;)]</span></span><br></pre></td></tr></table></figure>
-<h3 id="checkpyc.py">checkpyc.py</h3>
-<p>检查pyc文件是否最新</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> importlib.util</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(importlib.util.MAGIC_NUMBER)</span><br><span class="line"><span class="comment"># b&#x27;B\r\r\n&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="cleanfuture.py">cleanfuture.py</h3>
-<p>修改py文件的future语法</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">errprint</span>(<span class="params">*args</span>):</span><br><span class="line">    strings = <span class="built_in">map</span>(<span class="built_in">str</span>, args)</span><br><span class="line">    msg = <span class="string">&#x27; &#x27;</span>.join(strings)</span><br><span class="line">    <span class="keyword">if</span> msg[-<span class="number">1</span>:] != <span class="string">&#x27;\n&#x27;</span>:</span><br><span class="line">        msg += <span class="string">&#x27;\n&#x27;</span></span><br><span class="line">    sys.stderr.write(msg)</span><br><span class="line">    </span><br><span class="line">errprint(<span class="string">&quot;Usage:&quot;</span>, __doc__)</span><br></pre></td></tr></table></figure>
-<h3 id="crlf.py">crlf.py</h3>
-<p>文件中CRLF用LF替换</p>
-<h3 id="lfcr.py">lfcr.py</h3>
-<p>文件中LF用CRLF替换</p>
-<p>迭代：<code>itertools.count()</code></p>
-<h3 id="reindent.py">reindent.py</h3>
-<p>py文件改为4空格缩进</p>
-<h3 id="untabify.py">untabify.py</h3>
-<p>同上</p>
-<h3 id="texi2html.py">texi2html.py</h3>
-<p>texi to html</p>
-<p><code>import keyword</code></p>
-<p><code>import webbrowser</code></p>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>Scripts</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python杂谈</title>
-    <url>/hexo_blog/2018/03/14/Python/Python%E6%9D%82%E8%B0%88/</url>
-    <content><![CDATA[<p>理解Python中的GIL(Global Interpreter
-Lock)，以及如何使用多线程和多进程，以及弱引用。</p>
-<span id="more"></span>
-<h2 id="gil">GIL</h2>
-<p>建议阅读文章</p>
-<p><a
-href="http://zhuoqiang.me/python-thread-gil-and-ctypes.html">python
-线程，GIL 和 ctypes</a></p>
-<p><a
-href="https://zhuanlan.zhihu.com/p/20953544">谈谈python的GIL、多线程、多进程</a></p>
-<p>GIL, <code>Global Interpreter Lock</code>，全局解释器锁</p>
-<p>产生原因: 防止多线程并发执行机器码的Mutex。</p>
-<p>好处:
-不需要考虑线程安全问题，在单线程的情况下更快(相对而言)，和C库结合时更加方便</p>
-<p>形象的比方:</p>
-<blockquote>
-<p>我们把整个进程空间看做一个车间，把线程看成是多条不相交的流水线，把线程控制流中的字节码看作是流水线上待处理的物品。Python
-解释器是工人，整个车间仅此一名。操作系统是一只上帝之手，会随时把工人从一条流水线调到另一条——这种“随时”是不由分说的，即不管处理完当前物品与否。</p>
-</blockquote>
-<blockquote>
-<p>若没有 GIL。假设工人正在流水线 A 处理 A1 物品，根据 A1
-的需要将房间温度（一个全局对象）调到了 20
-度。这时上帝之手发动了，工人被调到流水线 B 处理 B1 物品，根据 B1
-的需要又将房间温度调到了 50 度。这时上帝之手又发动了，工人又调回 A
-继续处理 A1。但此时 A1 暴露在了 50 度的环境中，安全问题就此产生了。</p>
-</blockquote>
-<blockquote>
-<p>而 GIL 相当于一条锁链，一旦工人开始处理某条流水线上的物品，GIL
-便会将工人和该流水线锁在一起。而被锁住的工人只会处理该流水线上的物品。就算突然被调到另一条流水线，他也不会干活，而是干等至重新调回原来的流水线。这样每个物品在被处理的过程中便总是能保证全局环境不会突变。</p>
-</blockquote>
-<p>在解释器解释python代码时，首先会获得这把锁，需要I/O操作就是释放这把锁。如果是纯计算程序，没有I/O操作，解释器会每隔100次操作释放一次锁（Python3.2之后使用固定时间(5ms)来释放）。</p>
-<p>GIL只会影响严重依赖CPU的程序（计算型）,
-所以运行Python的时候，同一时间只能执行一个线程(单核CPU下的多线程都只是并发)。运行Python程序就只能占用一个核的CPU资源，哪怕使用了
-<code>threading</code>多线程也是一样。</p>
-<p>CPU密集性代码(循环)：不适用多线程。不断触发GIL，线程来回切换，消耗资源</p>
-<p>IO密集型代码(文件处理，网路爬虫)：多线程能够提高效率。在一个线程等待的同时，切换到另一个线程，不浪费CPU资源从而提升效率。</p>
-<p>多核多线程比单核多线程更差，单核下多线程每次释放GIL，唤醒的线程都能获取GIL锁，可以无缝执行。但是多核下，一个CPU释放完GIL之后，所有CPU的线程又开始竞争，可能会被原来的CPU获取，导致其他CPU线程唤醒之后必须等待到下一次释放。这样造成的结果称为线程颠簸(thrashing),
-效率会更低下。</p>
-<p>以上，如果Python需要利用多核CPU，就使用多进程(<code>multiprocessing</code>)，各自拥有独立的GIL，互不干扰，从而达到并行执行。还有一种方案就是使用C进行拓展，手动释放GIL，这里不做介绍。</p>
-<p>参考代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-   </span></span><br><span class="line"><span class="keyword">from</span> time <span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread</span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Process</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_threads</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Thread(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_processes</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Process(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">test</span>(<span class="params">target, number=<span class="number">10</span>, spawner=spawn_n_threads</span>):</span><br><span class="line">    <span class="keyword">for</span> n <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5</span>):</span><br><span class="line">        start_time = time()</span><br><span class="line">        <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(number):</span><br><span class="line">            spawner(n, target)</span><br><span class="line">        end_time = time()</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Time elapsed with &#123;&#125; branch(es): &#123;:.6f&#125; sec(s)&quot;</span>.<span class="built_in">format</span>(n, end_time - start_time))</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fib</span>():</span><br><span class="line">    a = b = <span class="number">1</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100000</span>):</span><br><span class="line">        a, b = b, a + b</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;threads:&quot;</span>)</span><br><span class="line">    test(fib)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;processes:&quot;</span>)</span><br><span class="line">    test(fib, spawner=spawn_n_processes)</span><br></pre></td></tr></table></figure>
-<p>P.S.</p>
-<ul>
-<li>并行：同时处理两个及以上的事件在同一时刻发生</li>
-<li>并发：同时处理两个及以上的事件在同一事件间隔内发生</li>
-</ul>
-<h2 id="弱引用">弱引用</h2>
-<p>推荐阅读</p>
-<p><a
-href="http://blog.soliloquize.org/2016/01/21/Python%E5%BC%B1%E5%BC%95%E7%94%A8%E7%9A%84%E4%BD%BF%E7%94%A8%E4%B8%8E%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9/">Python弱引用的使用与注意事项</a></p>
-<p>弱引用与强引用相对，是指不能确保其引用的对象不会被垃圾回收器回收的引用。若一个对象若只被弱引用所引用，则被认为是不可访问（或弱可访问）的，病因此可能在任何时刻被回收。主要作用就是减少循环引用，减少内存中不必要的对象存在的数量。</p>
-<p>Python用
-<code>weakref</code>模块创建对象的弱引用，当引用计数为0或只存在对象的弱引用时将回收这个对象。</p>
-<p>内存管理时使用，cache编程，垃圾回收(GC)</p>
-<h2 id="while-1与while-true">while 1与while True</h2>
-<p><a
-href="http://www.pythoner.com/356.html">Python天坑系列（一）：while
-1比while True更快？</a></p>
-<p>在python2中while 1会比while True更快</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_one</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_true</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    w1_time, wt_time = <span class="number">0</span>, <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        w1 = timeit.timeit(while_one, <span class="string">&quot;from __main__ import while_one&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        w1_time += w1</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        wt = timeit.timeit(while_true, <span class="string">&quot;from __main__ import while_true&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        wt_time += wt</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;while one: %s\nwhile_true: %s&quot;</span> % (w1_time, wt_time))</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 9.27971386909</span><br><span class="line">while_true: 13.5762348175</span><br><span class="line"><span class="comment"># python3的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 14.19957690299998</span><br><span class="line">while_true: 13.785004297995329</span><br></pre></td></tr></table></figure>
-<p>原因：python2中True/False不是关键字，python3添加了True/False做为关键字。每次在循环的时候，True/False需要被检查，所以while
-1会更快</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Python <span class="number">2.7</span><span class="number">.14</span> (default, Sep <span class="number">23</span> <span class="number">2017</span>, <span class="number">22</span>:06:<span class="number">14</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux2</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;exec&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;print&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br><span class="line"></span><br><span class="line">Python <span class="number">3.6</span><span class="number">.3</span> (default, Oct  <span class="number">3</span> <span class="number">2017</span>, <span class="number">21</span>:<span class="number">45</span>:<span class="number">48</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;False&#x27;</span>, <span class="string">&#x27;None&#x27;</span>, <span class="string">&#x27;True&#x27;</span>, <span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;nonlocal&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br></pre></td></tr></table></figure>
-<h2 id="if-x-true-与-if-x">if x == True 与 if x</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x_eq_true</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x == <span class="literal">True</span>:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    if1 = timeit.timeit(if_x_eq_true, <span class="string">&quot;from __main__ import if_x_eq_true&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line">    if2 = timeit.timeit(if_x, <span class="string">&quot;from __main__ import if_x&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;if_x_eq_true: %s\nif_x: %s&quot;</span> % (if1, if2))</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line">if_x_eq_true: 0.118808984756</span><br><span class="line">if_x: 0.0920550823212</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line">if_x_eq_true: 0.0951261969985353</span><br><span class="line">if_x: 0.07899275699855934</span><br></pre></td></tr></table></figure>
-<p>不管python2还是python3, <code>if x:</code>从各方面来说都会优于
-<code>if x == True</code>。后者会比前者多出一个检查True值和比较的操作。</p>
-<h2 id="type与object">type与object</h2>
-<p><a href="https://www.zhihu.com/question/38791962">Python 的 type 和
-object 之间是怎么一种关系？</a></p>
-<h2
-id="深刻理解python中的元类metaclass">深刻理解Python中的元类(metaclass)</h2>
-<p><a
-href="http://blog.jobbole.com/21351/">深刻理解Python中的元类(metaclass)</a></p>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>GIL</tag>
-        <tag>元类</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python 自动导入包</title>
-    <url>/hexo_blog/2023/12/03/Python/Python%E8%87%AA%E5%8A%A8%E5%AF%BC%E5%85%A5%E5%8C%85/</url>
-    <content><![CDATA[<p>在一个文件夹中，有很多个 Python
-文件，每个文件都有若干重名函数，需要一一导入。这个时候，可以用 Python
-的自动导入包来实现。</p>
-<span id="more"></span>
-<h2 id="需求描述">需求描述</h2>
-<p>假设有一个文件夹</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">func/</span><br><span class="line">├── __init__.py</span><br><span class="line">├── func1.py</span><br><span class="line">├── func2.py</span><br><span class="line">├── func3.py</span><br></pre></td></tr></table></figure>
-<p>每个文件都有一个重名函数，比如</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># func1.py</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">foo</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;foo&quot;</span>)</span><br></pre></td></tr></table></figure>
-<p>这不仅导致 <code>import</code> 代码不够简洁，且在新增文件时，还得在
-<code>__init__.py</code> 中添加 <code>import</code>
-语句。甚至在调用处也得增加代码来处理。</p>
-<h2 id="解决方案">解决方案</h2>
-<p>Python 的自动导入包可以解决这个问题。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> glob</span><br><span class="line"><span class="keyword">import</span> importlib</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取当前文件夹下所有的 .py 文件</span></span><br><span class="line">files = glob.glob(os.path.join(os.path.dirname(__file__), <span class="string">&quot;*.py&quot;</span>))</span><br><span class="line">func_dict = &#123;&#125;</span><br><span class="line"><span class="keyword">for</span> file_ <span class="keyword">in</span> files:</span><br><span class="line">    func_name = os.path.basename(file_)[:-<span class="number">3</span>]</span><br><span class="line">    load_module = importlib.import_module(<span class="string">f&quot;func.<span class="subst">&#123;func_name&#125;</span>&quot;</span>)</span><br><span class="line">    <span class="built_in">globals</span>()[load_module.__name__] = <span class="built_in">getattr</span>(load_module, <span class="string">&quot;foo&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将其放到一个字典中，以按需处理，</span></span><br><span class="line">    func_dict[func_name] = <span class="built_in">globals</span>()[load_module.__name__]</span><br></pre></td></tr></table></figure>
-<p>这样，不仅不需要在 <code>func/__init__.py</code> 中添加
-<code>import</code>
-语句，也不需要在调用处添加额外的代码。甚至，还要根据需求，从字典中调用对应函数。当然，这需要提前进行约定，比如全部放到<code>func</code>文件夹下，且都用<code>foo</code>这个名字作为函数名。</p>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>importlib</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python装饰器</title>
-    <url>/hexo_blog/2018/01/30/Python/Python%E8%A3%85%E9%A5%B0%E5%99%A8/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Python的装饰器是什么和一些Python内置装饰器的使用</p>
-<span id="more"></span>
-<p>参考文章:</p>
-<p><a
-href="https://www.zhihu.com/question/26930016">如何理解Python装饰器？</a></p>
-<p><a
-href="https://www.cnblogs.com/cicaday/p/python-decorator.html">详解Python的装饰器</a></p>
-<h2 id="装饰器是什么">装饰器是什么</h2>
-<p>python的装饰器本身就是一个函数，使用它可以在某些场景下简化python代码，增加一些额外功能。我的理解是，python装饰器可以帮助同时多个函数做一样的事情（比如打印日志），而避免了每个函数都需要额外加上一样的代码。</p>
-<p>举个例子，假设我们有两个函数</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>以上一段代码片段很容易理解，运行结果就是两行输出。那么问题来了，如果这个时候我需要在每个函数里面加上一个输出，比如<code>print("this is func")</code>。很简单，我们想到了第一种解决方案就是同时在两个函数里面加上这么一行。</p>
-<p>但是如果我们需要在很多个函数在上很多个这种重复代码，我们这么做就很浪费时间了，而且容易出错。这个时候我们想到了第二种解决方案。定义一个新的函数，在每次使用函数之前，先调用新函数。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">    func()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_print(func_one)</span><br><span class="line">    func_print(func_two)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>感觉不错，但是如果这样无疑改变了原来的逻辑结构，且每次需要传递函数给<code>func_print</code>。下面就轮到我们的主角登场了——装饰器。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one = func_print(func_one)</span><br><span class="line">    func_two = func_print(func_two)</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>这样也能达到我们的想要的效果，但是比之前的实现方式更难看懂。而且调用起来似乎更复杂了。</p>
-<p>第一个问题：更难看懂</p>
-<p>可能是对<code>*args</code>、<code>**kwargs</code>的参数理解不是很好。简单解释一下就是说,<code>*args</code>可以理解为一个<code>list</code>，接收多个参数。<code>**kwargs</code>可以理解为一个<code>dict</code>，接收多个<code>key</code>和<code>value</code>。接收的参数数量不定，可以没有，也可以很多个。再具体一点可以百度看看。</p>
-<p>第二个问题：调用更复杂</p>
-<p>其实上面的例子其实调用装饰器的一种方式，<code>python</code>有<code>@</code>的这个语法糖，所以一般情况下，我们是像下面一样调用的。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>好的，简单的装饰器调用基本就是这样，难度进阶，如何使用带参数的装饰器。换句话就说，如果我们需要根据每个函数自定义一些东西，如何传递参数给装饰器。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func_name</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">decorator</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">        <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;this is %s&quot;</span> % func_name)</span><br><span class="line">            <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">        <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> decorator</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params"><span class="string">&quot;one&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params">func_name=<span class="string">&quot;two&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>观察<code>func_print</code>部分代码，与之前的不同在于多加了一个<code>decorator</code>函数封装了<code>wrapper</code>函数。我的理解是这样的：<code>func_print</code>作用是装饰器的名字和接收外部的参数，<code>decorator</code>接收外部的函数，<code>wrapper</code>返回结果给函数。<code>decorator</code>翻译为中文是<strong>装饰</strong>。<code>wrapper</code>翻译过来是<strong>包装</strong>。<code>decorator</code>装饰函数，<code>wrapper</code>包装函数。</p>
-<p>当然，<code>decorator</code>和<code>wrapper</code>这两个函数名字可以自定义，在这里为了好理解所以用了这两个名字</p>
-<h2 id="内置装饰器">内置装饰器</h2>
-<h3 id="property"><code>@property</code></h3>
-<p>如果要了解使用这个装饰器，就需要知道在不用装饰器的情况下，写一个属性。这里使用python自带的一个例子。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">getx</span>(<span class="params">self</span>): <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">setx</span>(<span class="params">self, value</span>): <span class="variable language_">self</span>._x = value</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">delx</span>(<span class="params">self</span>): <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br><span class="line">    x = <span class="built_in">property</span>(getx, setx, delx, <span class="string">&quot;I&#x27;m the &#x27;x&#x27; property.&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>上面就是定义一个python属性的标准写法，下面使用<code>@property</code>来定义属性</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="string">&quot;I am the &#x27;x&#x27; property.&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line"><span class="meta">    @x.setter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>._x = value</span><br><span class="line"><span class="meta">    @x.deleter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br></pre></td></tr></table></figure>
-<p>这样有什么好处呢，原来的定义，属性访问并没有做限制。使用<code>@property</code>可以使<code>x</code>这个属性更加安全，从获取、设置必须与属性名一致。</p>
-<p>Emmmm.....我的理解就是加上<code>@property</code>的话，定义属性会更加安全更加优雅。毕竟是面向对象了，天知道你的用户会对你的属性做一些什么奇奇怪怪的事情。</p>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>Decorators</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python魔术方法</title>
-    <url>/hexo_blog/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Python的魔术方法</p>
-<span id="more"></span>
-<p>参考文章:</p>
-<p><a
-href="http://pycoders-weekly-chinese.readthedocs.io/en/latest/issue6/a-guide-to-pythons-magic-methods.html">Python
-魔术方法指南</a></p>
-<p>魔术方法，顾名思义是一种可以给对象(类)增加魔法的特殊方法，它们的表示方法一般是用双下划线包围(如<code>__init__</code>)</p>
-<h2 id="对象实例化">对象实例化</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> os.path <span class="keyword">import</span> join</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">FileObject</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;给文件对象进行包装从而确认在删除时文件流关闭&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, filepath=<span class="string">&#x27;~&#x27;</span>, filename=<span class="string">&#x27;sample.txt&#x27;</span></span>):</span><br><span class="line">        <span class="comment">#读写模式打开一个文件</span></span><br><span class="line">        <span class="variable language_">self</span>.file = <span class="built_in">open</span>(join(filepath, filename), <span class="string">&#x27;r+&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__del__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.file.close()</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.file</span><br></pre></td></tr></table></figure>
-<ul>
-<li><p><code>__new__(cls, [...])</code>:
-在一个对象实例化调用的第一个方法，第一个参数是类，剩下的参数传递给<code>__init__</code></p></li>
-<li><p><code>__init__(self, [...])</code>:
-类的初始化方法。当构造函数被调用的时候的任何参数够会传给它</p></li>
-<li><p><code>__del__(self)</code>:
-<code>__new__</code>和<code>__init__</code>为对象的构造器，<code>__del__</code>为对象的析构器。当一个对象进行过垃圾回收时候的行为就是这个方法。当一个对象在删除的时候需要更多的清洁工作，就需要使用这个方法</p></li>
-</ul>
-<h2 id="比较">比较</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Word</span>(<span class="title class_ inherited__">str</span>):</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;存储单词的类，定义比较单词的几种方法&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__new__</span>(<span class="params">cls, word</span>):</span><br><span class="line">        <span class="comment"># 注意我们必须要用到__new__方法，因为str是不可变类型</span></span><br><span class="line">        <span class="comment"># 所以我们必须在创建的时候将它初始化</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">&#x27; &#x27;</span> <span class="keyword">in</span> word:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;Value contains spaces. Truncating to first space.&quot;</span>)</span><br><span class="line">            word = word[:word.index(<span class="string">&#x27; &#x27;</span>)] <span class="comment">#单词是第一个空格之前的所有字符</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">str</span>.__new__(cls, word)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__gt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__lt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__ge__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt;= <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__le__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt;= <span class="built_in">len</span>(other)</span><br></pre></td></tr></table></figure>
-<ul>
-<li><p><code>__cmp__(self, other)</code>:
-用于比较的魔术方法，实现了所有的比较符号。如果<code>self</code> &lt;
-<code>other</code>返回一个负数；如果<code>self</code> ==
-<code>other</code>返回０；如果<code>self</code> &gt;
-<code>other</code>返回正数。一般情况下是分别定义每一个比较符号。如果想实现所有的比较符号，就需要调用这个方法。</p></li>
-<li><p><code>__eq__(self, other)</code>: 等号</p></li>
-<li><p><code>__ne__(self, other)</code>:　不等号</p></li>
-<li><p><code>__lt__(self, other)</code>:　小于</p></li>
-<li><p><code>__gt__(self, other)</code>:　大于</p></li>
-<li><p><code>__le__(self, other)</code>:　小于等于</p></li>
-<li><p><code>__ge__(self, other)</code>:　大于等于</p></li>
-</ul>
-<h2 id="一元操作符和函数">一元操作符和函数</h2>
-<ul>
-<li><code>__pos__(self)</code>：实现正号的特性</li>
-<li><code>__neg__(self)</code>：实现负号的特性</li>
-<li><code>__abs__(self)</code>：abs, 绝对值</li>
-<li><code>__invert__(self)</code>：实现~的特性</li>
-</ul>
-<h2 id="普通算数操作符">普通算数操作符</h2>
-<ul>
-<li><code>__add__(self, other)</code>: 实现加法</li>
-<li><code>__sub__(self, other)</code>: 实现减法</li>
-<li><code>__mul__(self, other)</code>: 实现乘法</li>
-<li><code>__floordiv__(self, other)</code>: 实现整除法</li>
-<li><code>__div__(self, other)</code>: 实现除法</li>
-<li><code>__mod__(self, other)</code>: 实现取模</li>
-<li><code>__pow__(self, other)</code>: 实现指数</li>
-<li><code>__lshift__(self, other)</code>: 实现&lt;&lt;</li>
-<li><code>__rshift__(self, other)</code>: 实现&gt;&gt;</li>
-<li><code>__and__(self, other)</code>: 实现&amp;</li>
-<li><code>__or__(self, other)</code>: 实现|</li>
-<li><code>__xor__(self, other)</code>: 实现^</li>
-</ul>
-<h2 id="其他">其他</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.name = value</span><br><span class="line">    <span class="comment">#每当属性被赋值的时候， ``__setattr__()`` 会被调用，这样就造成了递归调用。</span></span><br><span class="line">    <span class="comment">#这意味这会调用 ``self.__setattr__(&#x27;name&#x27;, value)`` ，每次方法会调用自己。这样会造成程序崩溃。</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.__dict__[name] = value  <span class="comment">#给类中的属性名分配值</span></span><br><span class="line">    <span class="comment">#定制特有属性</span></span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">AccessCounter</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;一个包含计数器的控制权限的类每当值被改变时计数器会加一&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, val</span>):</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="number">0</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;value&#x27;</span>, val)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">    <span class="comment">#如果你不想让其他属性被访问的话，那么可以抛出 AttributeError(name) 异常</span></span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(name, value)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delattr__</span>(<span class="params">self, name</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__delattr__(name)]</span><br></pre></td></tr></table></figure>
-<ul>
-<li><p><code>__str__(self)</code>: 返回值，人类可读</p></li>
-<li><p><code>__repr__(self)</code>: 返回值，机器可读</p></li>
-<li><p><code>__unicode__(self)</code>: 返回unicode()</p></li>
-<li><p><code>__hash__(self)</code>: hash()返回一个整形</p></li>
-<li><p><code>__nonzero__(self)</code>: bool()返回的值</p></li>
-<li><p><code>__getattr__(self, name)</code>:
-当用户试图获取一个不存在（或不建议的属性）的属性时，给出一些警告</p></li>
-<li><p><code>__setattr__(self, name, value)</code>:
-无论属性是否存在，都允许你定义对属性的赋值行为</p></li>
-<li><p><code>__delattr__(self, name)</code>： 删除一个属性</p></li>
-</ul>
-<h2 id="封装附加魔术方法">封装附加魔术方法</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">FunctionalList</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;一个封装了一些附加魔术方法比如 head, tail, init, last, drop, 和take的列表类。</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, values=<span class="literal">None</span></span>):</span><br><span class="line">        <span class="keyword">if</span> values <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = []</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = values</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__len__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__getitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="comment">#如果键的类型或者值无效，列表值将会抛出错误</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setitem__</span>(<span class="params">self, key, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values[key] = value</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__iter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">iter</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__reversed__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">reversed</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">append</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values.append(value)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">head</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tail</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">1</span>:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">init</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回一直到末尾的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">last</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回末尾元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">drop</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回除前n个外的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[n:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">take</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回前n个元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:n]</span><br></pre></td></tr></table></figure>
-<h2 id="反射">反射</h2>
-<ul>
-<li><code>__instancecheck__(self, instance)</code>:
-检查一个实例是否为定义类的实例</li>
-<li><code>__subclasscheck__(self, subclass)</code>:
-检查一个类是否为定义类的子类</li>
-</ul>
-<h2 id="更改状态">更改状态</h2>
-<ul>
-<li><code>__call__(self, [...])</code>:
-经常改变状态的时候，调用这个方法是一种直接和优雅的方法</li>
-</ul>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Entity</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;调用实体来改变实体的位置。&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, size, x, y</span>):</span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br><span class="line">        <span class="variable language_">self</span>.size = size</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__call__</span>(<span class="params">self, x, y</span>):</span><br><span class="line">        <span class="string">&#x27;&#x27;&#x27;改变实体的位置&#x27;&#x27;&#x27;</span></span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br></pre></td></tr></table></figure>
-<h2 id="会话管理">会话管理</h2>
-<p>使用with语句来调用</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Closer</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;通过with语句和一个close方法来关闭一个对象的会话管理器&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, obj</span>):</span><br><span class="line">        <span class="variable language_">self</span>.obj = obj</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__enter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.obj <span class="comment"># bound to target</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__exit__</span>(<span class="params">self, exception_type, exception_val, trace</span>):</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="variable language_">self</span>.obj.close()</span><br><span class="line">        <span class="keyword">except</span> AttributeError: <span class="comment"># obj isn&#x27;t closable</span></span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&#x27;Not closable.&#x27;</span>)</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">True</span> <span class="comment"># exception handled successfully</span></span><br></pre></td></tr></table></figure>
-<h2 id="slots__"><code>__slots__</code></h2>
-<p>限制class的属性，但不限制class的子类的属性，除非子类也使用<code>__slots__</code></p>
-<p>例子：</p>
-<p>使用<code>__slots__</code>会比不使用，减少40%~50%的内存</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 不使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  __slots__ = [<span class="string">&#x27;name&#x27;</span>, <span class="string">&#x27;identifier&#x27;</span>]</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>Magic Methods</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>多进程嵌套内存泄漏问题记录</title>
-    <url>/hexo_blog/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>记一次多进程嵌套发现的内存泄漏问题，以及解决方案。</p>
-<span id="more"></span>
-<h3 id="问题代码">问题代码</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> multiprocessing</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ProcessPoolExecutor</span><br><span class="line"><span class="keyword">import</span> concurrent.futures</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Any</span></span><br><span class="line"><span class="keyword">import</span> gc</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Foo</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.executor = ProcessPoolExecutor(max_workers=<span class="number">10</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>))</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">fun</span>(<span class="params">arg</span>):</span><br><span class="line">        <span class="keyword">raise</span> concurrent.futures.process._RemoteTraceback(<span class="string">&quot;error&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">        x = []</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">            res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="keyword">with</span> ProcessPoolExecutor(max_workers=<span class="number">2</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>)) <span class="keyword">as</span> e:</span><br><span class="line">        futures = e.submit(predict)</span><br><span class="line">        futures.result()</span><br></pre></td></tr></table></figure>
-<p>肉眼观察 乍一看这段代码是没有任何问题的，在 submit 之后，获取 result
-，并且用 try ... except ... 来捕获可能出现的任何情况。尤其是，当不使用
-main 函数中的进程池时，直接普通的使用 predict
-函数，这段代码不会引发任何问题，可以正常的退出。</p>
-<p>但为什么改成多进程时，这段代码就无法正常退出了呢？程序无法正常退出往往是因为程序某些对象没有完全被释放干净，但这里并没有使用特殊的函数却依旧引发了该问题，所以需要借助工具来进行简单的分析。</p>
-<h3 id="分析">分析</h3>
-<p>garbage collector python 的垃圾回收机制主要使用引用计数（reference
-counting）来跟踪和回收垃圾。大部分情况下，Python
-能够在对象结束执行后，引用计数会变为
-0，使得最后能够被回收。这里程序退出不了的原因是某些对象没发自动退出，所以在
-predict 函数中，观察变量 f 和 f.executor 的引用计数</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
-<p>程序的输出，对象 f 的计数为 5，而 f.executor 的计数为 2。为什么是
-2，因此在最外层还有一个进程数量为 2 的进程池没有执行完。而 f 等于 4
-而不等于 f.executor 这就不符合预期，比正常数值多 3。这个 3 是因为循环了
-3 次，每次都引发了异常。</p>
-<p>进一步地，观察 f 中什么对象没有被清理。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(gc.get_referrers(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
-<p>此时输出会多一行，形如</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">[&lt;frame at 0x10c266240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0440, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;]</span><br></pre></td></tr></table></figure>
-<p>这表明程序在何处的对象没有被处理，可以定位到函数 call 的 return
-位置。</p>
-<p>为什么？ 在获取 result 时，如果遇到异常，则中间变量 future
-无法被回收。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">result</span>(<span class="params">self, timeout=<span class="literal">None</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;Return the result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Args:</span></span><br><span class="line"><span class="string">        timeout: The number of seconds to wait for the result if the future</span></span><br><span class="line"><span class="string">            isn&#x27;t done. If None, then there is no limit on the wait time.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Returns:</span></span><br><span class="line"><span class="string">        The result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Raises:</span></span><br><span class="line"><span class="string">        CancelledError: If the future was cancelled.</span></span><br><span class="line"><span class="string">        TimeoutError: If the future didn&#x27;t finish executing before the given</span></span><br><span class="line"><span class="string">            timeout.</span></span><br><span class="line"><span class="string">        Exception: If the call raised then that exception will be raised.</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">with</span> <span class="variable language_">self</span>._condition:</span><br><span class="line">            <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line"></span><br><span class="line">    <span class="variable language_">self</span>._condition.wait(timeout)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">raise</span> TimeoutError()</span><br><span class="line">    <span class="keyword">finally</span>:</span><br><span class="line">        <span class="comment"># Break a reference cycle with the exception in self._exception</span></span><br><span class="line">        <span class="variable language_">self</span> = <span class="literal">None</span></span><br></pre></td></tr></table></figure>
-<h3 id="解决方案">解决方案</h3>
-<h4 id="消除中间变量-future">消除中间变量 future</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 改为</span></span><br><span class="line">res = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i).result(timeout=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<h4 id="手动-gc">手动 GC</h4>
-<p>在调用完 Foo 之后，使用 gc.collect()</p>
-<h4 id="主动关闭进程池">主动关闭进程池</h4>
-<p>在调用完 Foo 之后，使用 f.executor.shutdown()</p>
-<h4 id="在-return-前-主动删除未释放的对象">在 return 前
-主动删除未释放的对象</h4>
-<p>如下所示，使用 future = None 也可以达到相似效果（让 Python
-自动调用垃圾回收机制）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">   x = []</span><br><span class="line">   <span class="keyword">try</span>:</span><br><span class="line">       future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">       res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">   <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">       <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">       <span class="keyword">del</span> future</span><br><span class="line">   <span class="keyword">return</span> x</span><br></pre></td></tr></table></figure>
-<h3 id="q-a">Q &amp; A</h3>
-<p>Q: 能不能在 Foo 中添加 <strong>del</strong> 函数解决这个问题 A: 在
-Python 3.8.16 可以，Python 3.9.6 不行。可以手动增加 <strong>del</strong>
-函数添加 print 函数，观察是否有对应的输出。</p>
-<p>Q: gc 显示为什么是在 return 这，而不是其他地方？ A: frame
-对象有一个属性叫作 f_lineno，它表示当前执行的代码行。当函数执行到 return
-语句时，f_lineno 会更新为 return 语句所在的行号。</p>
-<p>这是因为在函数返回值之后，控制权会返回到调用该函数的代码。此时函数的上下文（包括它的局部变量、执行状态等）会被丢弃，栈帧也会被弹出调用栈。这个过程称为函数的退出或返回。所以，return
-语句所在的行是函数执行的最后一行。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 最后将其更新到输出矩阵中</span></span><br><span class="line">output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br></pre></td></tr></table></figure>
+<p>完整代码如下（这里没有实现完整的attention，仅仅是一个矩阵乘法 +
+softmax 计算）：</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">        __shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        __shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = value;</span><br><span class="line">        __syncthreads();</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax 的分母</span></span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line"></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 计算每个 row  normalizer_term之和</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax</span></span><br><span class="line">        output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 这里偷懒没有换名字</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores&#125;;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>测试代码如下</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> mulsoftmax</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">seed = <span class="number">42</span></span><br><span class="line">torch.manual_seed(seed)</span><br><span class="line">torch.cuda.manual_seed(seed)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_py_softmax</span>(<span class="params">x, dim</span>):</span><br><span class="line">    e = torch.exp(x)</span><br><span class="line">    s = torch.<span class="built_in">sum</span>(e, dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">return</span> e / s</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_mulsoft</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="comment"># print(q@k.T)</span></span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_mulsoft(q, k, v)</span><br><span class="line">    cpp_values = mulsoftmax.forward(q, k, v)[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">loop=<span class="number">100</span></span>):</span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_mulsoft(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: mulsoftmax.forward(q, k, v)[<span class="number">0</span>], number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">16</span>, <span class="number">40</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="comment"># 先检查结果是否正确</span></span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    q, k, v = torch.rand(size=(m, n)), torch.rand(size=(m, n)), torch.rand(size=(m, n))</span><br><span class="line">    <span class="comment"># 循环1w次，对比性能差距</span></span><br><span class="line">    compare_time(<span class="number">10000</span>)</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.5136</td>
+<td>0.2909</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.6143</td>
+<td>0.3322</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.7300</td>
+<td>0.3608</td>
+</tr>
+</tbody>
+</table>
+<h3 id="其他实现">其他实现</h3>
+<h4
+id="二维矩阵转一维矩阵实现矩阵乘法">二维矩阵转一维矩阵，实现矩阵乘法</h4>
+<p>在某些情况下，为了降低空间复杂度，会把二维矩阵展开为一维矩阵，再进行矩阵乘法。这里在这个基础上，再实现了
+softmax
+的计算。其本质上是把二维矩阵转为一维矩阵，再进行矩阵乘法。原来的<code>input1[row][k]</code>
+-&gt; <code>input1[row * K + k]</code>，<code>input2[k][col]</code>
+-&gt; <code>input2[k * N + col]</code>。但这里暂时不好实现 softmax
+融合，因为 softmax
+需要计算每行的最大值，这里的一维矩阵无法直接计算每行的最大值。所以这里偷懒先使用判断的方法，如果是最后一列，则计算
+softmax。因此，导致算的速度会比较慢。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line">#DEFINE BLOCK_SIZE <span class="number">256</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_vector_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N, <span class="type">const</span> <span class="type">int</span> K</span></span></span><br><span class="line"><span class="params"><span class="function">                                       )</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line">    <span class="type">int</span> col = index % N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M &amp;&amp; col &lt; N) &#123;</span><br><span class="line">        <span class="type">float</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; K; ++k) &#123;</span><br><span class="line">            value += input1[row * K + k] * input2[k * N + col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row * N + col] = value;</span><br><span class="line">        <span class="keyword">if</span> (col == N - <span class="number">1</span>) &#123;</span><br><span class="line">            <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">            <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                old_row_max = row_max;</span><br><span class="line">                row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">                normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">            &#125;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">matmul_vector</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> &#123;output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;), output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;)&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">matmul_vector</span>(q, k);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.4239</td>
+<td>0.4907</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.5069</td>
+<td>0.4388</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.5462</td>
+<td>0.5799</td>
+</tr>
+</tbody>
+</table>
+<p>尽管用了一种比较笨的方法，但是速度实际上与原生的 pytorch
+相差无几。</p>
+<h4 id="单独实现-softmax">单独实现 softmax</h4>
+<p>相较于非算子融合的写法，这里实现了一维矩阵的
+softmax。相当于在计算矩阵乘法后再算 softmax。比较朴素的写法。。。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">softmax_kernel</span><span class="params">(torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                 <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M) &#123;</span><br><span class="line">        <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">        <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">        <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            old_row_max = row_max;</span><br><span class="line">            row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">            normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_softmax_vector_softmax</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE_VECTOR)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 普通一维的矩阵乘法</span></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="built_in">cudaDeviceSynchronize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(output_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;);</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.4239</td>
+<td>1.1874</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.5069</td>
+<td>1.1101</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.5462</td>
+<td>1.3238</td>
+</tr>
+</tbody>
+</table>
+<p>对比发现，这里的cuda还是会比pytorch的慢一些，这是因为没有用自带的softmax，且没有使用算子融合。所以就算速度会更慢。</p>
+<h2 id="参考">参考</h2>
+<p>[1] <a
+href="https://pytorch.org/tutorials/advanced/cpp_extension.html">https://pytorch.org/tutorials/advanced/cpp_extension.html</a>。</p>
+<p>[2] <a
+href="https://github.com/ELS-RD/kernl/blob/main/tutorial/1%20-%20tiled%20matmul.ipynb">Tiled
+matmul</a></p>
+<p>[3] <a
+href="https://github.com/ELS-RD/kernl/blob/main/tutorial/3%20-%20online%20softmax.ipynb">Online
+softmax</a></p>
+<p>[4] <a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
+calculation for softmax</a></p>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>DeepLearning</category>
       </categories>
       <tags>
-        <tag>多进程</tag>
-        <tag>内存泄漏</tag>
+        <tag>CUDA</tag>
+        <tag>PyTorch</tag>
+        <tag>Cpp</tag>
       </tags>
   </entry>
   <entry>
-    <title>拉格朗日乘子法：从函数的求导到不等式约束的求解</title>
-    <url>/hexo_blog/2018/04/30/%E6%95%B0%E5%AD%A6/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>本篇文章简单梳理了关于拉格朗日乘子法的相关问题及一些相关概念的介绍。</p>
+    <title>torch中的scheduler</title>
+    <url>/hexo_blog/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>PyTorch学习率调整过程中版本问题引发的不同结果解析</p>
 <span id="more"></span>
-<h3 id="基本概念">基本概念</h3>
-<p>拉格朗日乘子法（Lagrange
-Multiplier）在某些地方又叫做拉格朗日乘数法。在求解最优化问题中，这个方法是最常用的方法之一。最优化问题往往抽象为某个函数，即求函数的最大值或最小值。求解最大值和最小值的求解可以相互转换（对偶）。对于求解最优化问题分为以下三种情况（难度依此递增）：</p>
-<h3 id="无约束条件">无约束条件</h3>
-<p>一般情况下，只需要对给出的函数进行求导，令其导函数为0的点可能就是极值点，将可能的结果分别代回原函数进行验证即可。</p>
-<p>假设有这样一个函数<span class="math inline">\(f(x)\)</span>，令<span
-class="math inline">\(g(x)=f^{&#39;}(x)\)</span>，<span
-class="math inline">\(g(x)=0\)</span>的解有<span
-class="math inline">\(x_1,x_2,x_3,\cdots\)</span>。求函数<span
-class="math inline">\(f(x)\)</span>的最优解就只需要分别将<span
-class="math inline">\(x_1,x_2,x_3,\cdots\)</span>代入<span
-class="math inline">\(f(x)\)</span>，比较大小，得到最优解<span
-class="math inline">\(x_i\)</span>。</p>
-<p>从这里开始引出<strong>拉格朗日乘子法</strong></p>
-<h3 id="等式约束条件">等式约束条件</h3>
-<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
-约束条件为$h_k(x) $, <span
-class="math inline">\(k=1,2,\cdots,l\)</span>，即要求<span
-class="math inline">\(f(x)\)</span>的最优解，但是有<span
-class="math inline">\(l\)</span>个<span
-class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
-<p><span class="math display">\[
-F(x, \lambda)=f(x)+\sum\limits_{k=1}^{l}\lambda_kh_k(x)
-\]</span></p>
-<p><span
-class="math inline">\(\lambda_k\)</span>表示约束条件的待定系数。之后解变量的偏导方程，如下:</p>
-<p><span class="math display">\[
-\frac{\partial F}{\partial x_k}=0, \frac{\partial F}{\partial
-\lambda_k}=0, k=1,2,\cdots, l
-\]</span></p>
-<p>求解得到了<span
-class="math inline">\(l+1\)</span>个方程，将结果代回原方程验证即可得到解。</p>
-<p>举例：</p>
-<p>给定一个椭圆<span
-class="math inline">\(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}=1\)</span>，求解<span
-class="math inline">\(f(x,y,z)=8xyz\)</span>的最大值。</p>
-<p>求解过程如下：</p>
-<ul>
-<li><p>通过拉格朗日乘子法的定义，得到如下方程</p>
-<p><span class="math display">\[
-\begin{array}{l}F(x,y,z,\lambda)&amp;=f(x,y,z)+\lambda\cdot h(x,y,z) \\
-&amp;=8xyz+\lambda\cdot(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1)
-\end{array}
-\]</span></p></li>
-<li><p>接下来对<span
-class="math inline">\(F(x,y,z,\lambda)\)</span>求偏导，分别得到</p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial x}=8yz+\frac{2\lambda
-x}{a^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial y}=8xz+\frac{2\lambda
-y}{b^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial z}=8xy+\frac{2\lambda
-z}{c^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial
-\lambda}=\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1=0
-\]</span></p></li>
-<li><p>联立前三个方程可以得到<span
-class="math inline">\(bx=ay\)</span>和<span
-class="math inline">\(az=cx\)</span>，代入最后一个方程，解得：</p>
-<p><span class="math display">\[
-x=\frac{\sqrt{3}}{3}a
-\]</span></p>
-<p><span class="math display">\[
-y=\frac{\sqrt{3}}{3}b
-\]</span></p>
-<p><span class="math display">\[
-z=\frac{\sqrt{3}}{3}c
-\]</span></p></li>
-<li><p>最后代入原方程，解得最大体积</p>
-<p><span class="math display">\[
-V_{max}=f(\frac{\sqrt{3}}{3}a,\frac{\sqrt{3}}{3}b,\frac{\sqrt{3}}{3}c)=\frac{8\sqrt{3}}{9}abc
-\]</span></p></li>
-</ul>
-<h3 id="不等式约束条件">不等式约束条件</h3>
-<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
-不等式约束条件为<span class="math inline">\(c_i(x)\)</span>,<span
-class="math inline">\(i=1,2,\cdots,k\)</span>，等式约束条件为$h_j(x) $,
-<span class="math inline">\(j=1,2,\cdots,l\)</span>，即要求<span
-class="math inline">\(f(x)\)</span>的最优解，但是有<span
-class="math inline">\(k\)</span>个<span
-class="math inline">\(g(x)\)</span>不等式和<span
-class="math inline">\(l\)</span>个<span
-class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
-<p><span class="math display">\[
-L(x,a,b)=f(x)+\sum\limits^{k}_{i=1}a_ic_i(x)+\sum\limits^{l}_{j=1}b_jh_j(x)
-\]</span></p>
-<p>这里用一个叫做KKT条件的方法，把所有的不等式约束、等式约束和目标函数全部写为一个式子<span
-class="math inline">\(L(a,b,x)=f(x)+a*c(x)+b*h(x)\)</span>。</p>
-<p>KKT条件是指最优解必须满足以下条件：</p>
-<ul>
-<li><span class="math inline">\(L(a,b,x)\)</span>求<span
-class="math inline">\(x\)</span>求导为0</li>
-<li><span class="math inline">\(h(x)=0\)</span></li>
-<li><span class="math inline">\(a*c(x)=0\)</span></li>
-</ul>
-<p>这样就可以将不等式约束转换为了等式约束，求<span
-class="math inline">\(L(a,b,x)\)</span>的偏导等于零即可求得最优参数。</p>
-<p><span class="math display">\[
-\min f(x)\Leftrightarrow \min\limits_x \max\limits_{a,b} L(x,a,b)
-\]</span></p>
-<p>经过对偶变换得：</p>
-<p><span class="math display">\[
-\max\limits_{a,b} \min L(x,a,b)
-\]</span></p>
-<p>因为<span class="math inline">\(c(x)&lt;0\)</span>，所以只有当<span
-class="math inline">\(a*c(x)=0\)</span>时，<span
-class="math inline">\(L(x,a,b)\)</span>才能取得最大值。以上就可以使用同等式约束的求解方法进行求解。</p>
-<h3 id="补充部分">补充部分</h3>
-<h4 id="定义不等式约束问题的标准写法">定义不等式约束问题的标准写法</h4>
-<p><span class="math inline">\(f(x), c_i(x),h_j(x)\)</span>是定义在<span
-class="math inline">\(R^n\)</span>上的连续可微函数，考虑约束最优化问题：</p>
-<p><span class="math inline">\(\min f(x)  \quad  x\in R^n\)</span></p>
-<p><span class="math inline">\(s.t. c_i(x) \leq 0 \quad i =
-1...k\)</span></p>
-<pre><code>$h_j(x)=0 \quad j = 1...l$</code></pre>
-<p>即有<span class="math inline">\(k\)</span>个不等式约束<span
-class="math inline">\(c_i(x)\)</span>和<span
-class="math inline">\(l\)</span>个等式约束<span
-class="math inline">\(h_j(x)\)</span></p>
-<h4 id="拉格朗日函数">拉格朗日函数</h4>
-<p><span class="math inline">\(L(x,\alpha,
-\beta)=f(x)+\sum^{k}_{i=1}\alpha_ic_i(x)+\sum^{l}_{j=1}\beta_jh_j(x)\)</span></p>
-<p>其中的<span class="math inline">\(\alpha_i\)</span>和<span
-class="math inline">\(\beta_j\)</span>，称为<strong>拉格朗日乘子</strong>，<span
-class="math inline">\(\alpha_i \geq 0\)</span></p>
-<h4 id="几何意义">几何意义</h4>
-<p>对于一个二维空间来说，求解最短路径的问题。将维度升高，变为对于多维空间来说，求解最短路径问题。</p>
-<p>比较直观的解释，爬山的目标是到山顶，根据梯度上升（下降）的方法来寻求优解，即爬到山顶。</p>
-<h4 id="对偶概念的简单介绍">对偶概念的简单介绍</h4>
-<p>一对一的一种方式，把一种概念、公理或数学结构转换为另一种概念、公理或数学结构。如果A的对偶是B，那么B的对偶是A。A的对偶有时候是A的本身。</p>
-<p>有界线性泛函数：复杂对象映射到<strong>简单对象</strong>上</p>
-<p>群表示：简单对象具有类似线性结构的<strong>简单结构</strong></p>
-<p>范畴论：映射箭头所构成的集合</p>
-<p>（像集，映射）——&gt;原像集</p>
-<p>对联可以称为一种对偶，上联与下联有对应的关系（相似和取逆）。对偶是有一定要有自然合理的对应关系（或者说能够吹出来的自然合理关系）。</p>
-<p>常见的自然、合理法则首推：相似（包括同构）和取逆（包括互反）</p>
-<p>常见对偶的例子：</p>
-<ol type="1">
-<li>奇数和偶数</li>
-<li>洛必达法则和Stolz 定理（相当于离散洛必达法则）</li>
-<li>Fourier 系数与 Fourier 变换（傅里叶）</li>
-</ol>
-<p>对偶可以理解为，一个问题不好理解转换为另一个简单方式的问题来理解，二者在本质上是相同的。</p>
-<p>比如：某个函数的最大化不方便求解，那么就求这个函数的最小化。</p>
-<p>未完待续……</p>
-<ol type="1">
-<li>罚函数</li>
-<li>广义乘子法</li>
-<li>KKT的求导证明</li>
-</ol>
+<p>代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">optimizer1 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler1 = torch.optim.lr_scheduler.MultiStepLR(optimizer1,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line">optimizer2 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler2 = torch.optim.lr_scheduler.MultiStepLR(optimizer2,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">15</span>):</span><br><span class="line">    exp_lr_scheduler1.step()</span><br><span class="line">    exp_lr_scheduler2.step(epoch)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;Epoch &#123;&#125;, lr1 &#123;&#125;, lr2 &#123;&#125;&#x27;</span>.<span class="built_in">format</span>(epoch,</span><br><span class="line">        optimizer1.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>],</span><br><span class="line">        optimizer2.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>]))</span><br></pre></td></tr></table></figure>
+<p>当torch版本为1.2.0时，输出如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
+<p>当torch版本为1.4.0时，输出如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
+<p>小结：<code>scheduler.step()</code>中的
+<code>epoch</code>参数由于版本问题会带来不同的作用效果。1.2.0版本是判断epoch是否在某个区间内，而1.4.0版本是会直接调整学习到达最后一个区间。</p>
 ]]></content>
       <categories>
-        <category>MachineLearning</category>
+        <category>Note</category>
       </categories>
       <tags>
-        <tag>数学</tag>
-        <tag>Optimization</tag>
-        <tag>拉格朗日乘子法</tag>
+        <tag>PyTorch</tag>
+        <tag>scheduler</tag>
       </tags>
   </entry>
   <entry>
-    <title>最优化算法学习记录</title>
-    <url>/hexo_blog/2021/02/03/%E6%95%B0%E5%AD%A6/%E6%9C%80%E4%BC%98%E5%8C%96%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>本文提供了优化问题的概述，包括背景，分类，以及线性，非线性和二次规划问题的求解方法。本文还讨论了全局优化，并提供了非凸函数和凸函数的例子，以及线性规划和二次规划的例子。</p>
+    <title>交叉熵的反向传播 Step by Step（PyTorch）</title>
+    <url>/hexo_blog/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</url>
+    <content><![CDATA[<p>本文从公式开始，一步步用PyTorch实现自定义的交叉熵损失函数，最后理解分布式损失函数</p>
 <span id="more"></span>
-<p>优化问题的最终目的就是求最小值（最大值转为求最小值）</p>
-<h3 id="简单划分">简单划分</h3>
-<ol type="1">
-<li>解析法</li>
-<li><strong>迭代法</strong>
-<ol type="1">
-<li>常规方法</li>
-<li>技巧
-<ol type="1">
-<li>切平面</li>
-<li>椭圆法</li>
-<li>内点法</li>
-</ol></li>
-</ol></li>
-</ol>
-<h3 id="background">Background</h3>
+<h2 id="step-1a-simple-case">Step 1：A Simple Case</h2>
+<p>以交叉熵为例，普通的二分类问题 <span class="math display">\[
+\text {Binary Cross Entropy} = -\left (y \log (\hat {y}) + (1-y) \log
+(1-\hat {y})\right)
+\]</span> 更常见的多分类问题 <span class="math display">\[
+\text {Multiclass Cross Entropy} = -\sum_{c=0}^{C} y_c \log (\hat {y}_c)
+\]</span> 其中，<span class="math inline">\(y\)</span>
+表示真实的标签值（通常是one-hot向量），<span
+class="math inline">\(\hat{y}\)</span> 表示模型的输出，<span
+class="math inline">\(C\)</span> 表示有多少类别需要分类</p>
+<p>在 PyTorch 中调用（本文使用的torch version==2.0.0）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">criterion = torch.nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])   <span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])                 <span class="comment"># y       Shape: bs</span></span><br><span class="line"></span><br><span class="line">loss = criterion(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
+<p>这里的<code>outputs</code>是模型的输出，<code>targets</code>是真实的标签值</p>
+<h2 id="step-2-手写">Step 2: 手写</h2>
+<p>实现<code>CrossEntropyLoss</code>需要两个条件</p>
 <ol type="1">
-<li>目标函数、约束、线性（非线性）</li>
-<li>凸优化问题（是否为凸问题）</li>
-<li>局部最优、全局最优</li>
-<li>矩阵求导（梯度），矩阵的一阶导数、二阶导数（Hessian）</li>
-<li>解析与迭代</li>
+<li>使用 PyTorch 写出损失函数的公式，以及对应的导数</li>
+<li>根据公式计算loss，并计算梯度（grad）进行反向传播</li>
 </ol>
-<h3 id="how">How</h3>
+<h3 id="公式">公式</h3>
+<p>从<span class="math inline">\(-\sum_{c=0}^{C} y_c \log (\hat
+{y}_c)\)</span>出发</p>
 <ol type="1">
-<li>目标函数、约束均为线性：单纯形法。大问题：内点法</li>
-<li>目标函数二次，约束线性：改进单纯形法。大问题：内点法</li>
-<li>凸问题：切割平面、椭球算法、<strong>内点法</strong></li>
-<li>目标函数存在多个局部最小值，全局优化算法（启发式算法/智能优化算法）</li>
-<li>非线性非凸问题（最难），梯度和海森信息（二阶导）-&gt;构造解析表达式</li>
-<li>无约束问题
+<li>由于模型输出并不是单纯的概率值，因此会对模型输出进行<code>softmax</code>计算，得到<span
+class="math inline">\(\hat{y}\)</span> <span
+class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></li>
+<li>再以简单的单分类任务来说，
 <ol type="1">
-<li>Levenberg-Marquardt</li>
-<li>牛顿算法</li>
-<li>拟牛顿</li>
-<li>最速下降</li>
-<li>Powell</li>
+<li><span class="math inline">\(y\)</span> = [1, 0, 0], <span
+class="math inline">\(\hat{y}\)</span> = [0.5, 0.2, 0.3] -&gt; log(<span
+class="math inline">\(\hat{y}\)</span>)=[-0.9398, -1.2398, -1.1398]</li>
+<li>计算时，只有c=0时，结果有值（-0.9398），其他类别时为0，无需计算</li>
+<li>因此，实际最终结果就是取出对应类别的 <span
+class="math inline">\(\hat{y}\)</span> 的log值</li>
 </ol></li>
-<li>线性约束：SQP、梯度投影法</li>
-<li>非线性约束：SQP、转为为无约束</li>
 </ol>
-<h3 id="优化问题分类">优化问题分类</h3>
-<ol type="1">
-<li><p>无约束问题（非线性最小二乘问题）</p>
-<ol type="1">
-<li>牛顿法</li>
-<li>逆牛顿法</li>
-<li>Levenberg-Marquardt算法</li>
-<li>Nelder-Mead 方法</li>
-<li>其他
-<ol type="1">
-<li>搜索方向和一维搜索法</li>
-</ol></li>
-</ol></li>
-<li><p>约束问题</p>
-<ol type="1">
-<li>规划问题
-<ol type="1">
-<li>线性规划（整数规划）
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># outputs.shape: bs, C</span></span><br><span class="line">    <span class="comment"># targets.shape: bs</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  <span class="comment"># 对预测结果进行softmax操作</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 使用索引选择对应类别的概率值，并使用负对数函数计算损失</span></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) <span class="comment"># 对每个计算取对应的target</span></span><br><span class="line">    <span class="keyword">return</span> loss.mean() <span class="comment"># 最终返回结果的平均值</span></span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])	<span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])	</span><br><span class="line"></span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
+<h4 id="梯度">梯度</h4>
 <ol type="1">
-<li>单纯形解法</li>
-</ol></li>
-<li>二次规划
+<li><p>结合前面的计算，每次计算loss时，实际上只有对应类别的计算是有效的，因此将其简化为<span
+class="math inline">\(-y_c\log(\hat{y}_c)\)</span>，而由于<span
+class="math inline">\(y_c=1\)</span>，所以其实只有<span
+class="math inline">\(-\log(\hat{y}_c)\)</span></p></li>
+<li><p>这个公式只需要计算log的导数。对于<span
+class="math inline">\(f(x)=\log(x)\)</span>，其导数<span
+class="math inline">\(f^{&#39;}(x)=\frac{1}{x}\)</span>。</p></li>
+<li><p>因此，这里的梯度很简单，如下所示，其中，<span
+class="math inline">\(y_c=1\)</span> <span class="math display">\[
+\frac {\partial \text {CrossEntropy}(y, \hat {y})}{\partial \hat {y}_c}
+= -\frac {y_c}{\hat {y}_c}=-\frac{1}{\hat{y}_c}
+\]</span></p></li>
+<li><p>而对于模型的输出而言，还有一个SoftMax计算。SoftMax的梯度有</p>
 <ol type="1">
-<li>改进单纯形法</li>
-</ol></li>
-</ol></li>
-<li>非线性优化的约束
+<li>当 <span class="math inline">\(i = j\)</span> 时：</li>
+</ol>
+<p><span class="math display">\[
+\frac {\partial y_i}{\partial x_i} = \frac {e^{x_i}\left (\sum_{k=1}^n
+e^{x_k}\right) - e^{x_i} e^{x_i}}{\left (\sum_{k=1}^n e^{x_k}\right)^2}
+= y_i (1-y_i)
+\]</span></p>
+<ol start="2" type="1">
+<li>当 <span class="math inline">\(i \neq j\)</span> 时：</li>
+</ol>
+<p><span class="math display">\[
+\frac {\partial y_i}{\partial x_j} = -\frac {e^{x_j} e^{x_i}}{\left
+(\sum_{k=1}^n e^{x_k}\right)^2} = -y_i y_j
+\]</span></p></li>
+<li><p>由于求导的链式法则，因此将SoftMax的导数乘以交叉熵的导数，就能得到模型输出的梯度。综上，对于模型的输出，交叉熵的梯度仅在目标标签上有效（≠0），所以最后的梯度计算为，</p></li>
+</ol>
+<p><span class="math display">\[-\frac{1}{\hat{y}_c}\times
+y_c(1-y_c)=y_c-1\]</span></p>
+<h3 id="autograd">autograd</h3>
+<p>在PyTorch内部有一套完善的求导机制。因此，仅需按照其规则进行书写即可实现前向传播、反向传播（链式求导）。如下所示，在forward填写loss的计算，backward中填写grad的计算即可</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">CrossEntropyLoss</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        <span class="comment"># outputs：模型的输出</span></span><br><span class="line">        <span class="comment"># targets: 真实标签</span></span><br><span class="line">        ...</span><br><span class="line">        ctx.save_for_backward(...) <span class="comment"># 需要在反向传播中使用的变量</span></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        <span class="comment"># grad_output: 反向传播的梯度</span></span><br><span class="line">        ... = ctx.saved_tensors</span><br><span class="line">        ...</span><br><span class="line">	    <span class="keyword">return</span> new_grad_output</span><br></pre></td></tr></table></figure>
+<h4 id="forward">Forward</h4>
+<p>这里基本就是把公式照抄过来，但需要额外把梯度计算时用到的变量存一下。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">    ctx.save_for_backward(outputs, targets)</span><br><span class="line">    <span class="keyword">return</span> loss.mean()</span><br></pre></td></tr></table></figure>
+<h4 id="backward">Backward</h4>
+<p>这里的<code>grad_output</code>
+表示梯度计算的输出。它是一个与前向传播函数的输出形状相同的张量。在反向传播过程中，我们将计算当前函数的导数，乘以
+<code>grad_output</code>
+作为输入。这个值代表了后续节点对当前节点的梯度贡献。</p>
+<p>通过将 <code>grad_output</code>
+乘以导数（也称为雅可比向量积），可以有效地传递梯度信息到较早的节点，从而实现自动微分。</p>
+<p>注意，<code>grad_output</code>
+的形状必须与函数的输出形状一致，否则会引发错误。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">    outputs, targets = ctx.saved_tensors</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    </span><br><span class="line">    grad_y_pred = outputs.clone()</span><br><span class="line">    grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 计算对应类别的梯度</span></span><br><span class="line">    grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">    </span><br><span class="line">    grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">    </span><br><span class="line">    <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br></pre></td></tr></table></figure>
+<h3 id="完整版">完整版</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyCrossEntropyLossFunction</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">        loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">        ctx.save_for_backward(outputs, targets)</span><br><span class="line">        <span class="keyword">return</span> loss.mean()</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        outputs, targets = ctx.saved_tensors</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        </span><br><span class="line">        grad_y_pred = outputs.clone()</span><br><span class="line">        grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 只需计算对应标签的梯度</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">        </span><br><span class="line">        <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br><span class="line">  </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="keyword">return</span> MyCrossEntropyLossFunction.apply(outputs, targets)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]], requires_grad=<span class="literal">True</span>)</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])</span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398, grad_fn=&lt;MyCrossEntropyLossFunctionBackward&gt;)</span></span><br></pre></td></tr></table></figure>
+<p>结合简单的双层MLP</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MLP</span>(torch.nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>().__init__()</span><br><span class="line">        <span class="variable language_">self</span>.l1 = torch.nn.Linear(<span class="number">5</span>, <span class="number">5</span>, bias=<span class="literal">False</span>)</span><br><span class="line">        <span class="variable language_">self</span>.l2 = torch.nn.Linear(<span class="number">5</span>, <span class="number">3</span>, bias=<span class="literal">False</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.l2(<span class="variable language_">self</span>.l1(x))</span><br><span class="line">    </span><br><span class="line">input_ = torch.rand((<span class="number">5</span>, <span class="number">5</span>))</span><br><span class="line">labels = torch.randint(<span class="number">0</span>, <span class="number">3</span>, size=(<span class="number">5</span>,))</span><br><span class="line">model1 = MLP()</span><br><span class="line">model2 = copy.deepcopy(model1)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;Before Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br><span class="line">criterion1 = torch.nn.CrossEntropyLoss()</span><br><span class="line">criterion2 = MyCrossEntropyLoss</span><br><span class="line"></span><br><span class="line">optim1 = torch.optim.Adam(model1.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line">optim2 = torch.optim.Adam(model2.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line"></span><br><span class="line">outputs1 = model1(copy.deepcopy(input_))</span><br><span class="line">loss1 = criterion1(outputs1, labels)</span><br><span class="line">optim1.zero_grad()</span><br><span class="line">loss1.backward()</span><br><span class="line">optim1.step()</span><br><span class="line"></span><br><span class="line">outputs2 = model2(copy.deepcopy(input_))</span><br><span class="line">loss2 = criterion2(outputs2, labels)</span><br><span class="line">optim2.zero_grad()</span><br><span class="line">loss2.backward()</span><br><span class="line">optim2.step()</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;After Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h2 id="step-3-分布式">Step 3: 分布式</h2>
+<p>模型训练速度提升+单卡放不下大模型，由于这两个需求，所以分布式计算中需要重写损失函数的forward和backward，以Megatron-LM的训练代码为例，如下所示。接下来将逐步拆解实现过程（忽略label_smoothing，将其看作0.0）。</p>
+<p><a
+href="https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/cross_entropy.py">cross_entropy.py</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> megatron.core.parallel_state <span class="keyword">import</span> (</span><br><span class="line">    get_tensor_model_parallel_group,</span><br><span class="line">    get_tensor_model_parallel_rank,</span><br><span class="line">    get_tensor_model_parallel_world_size,</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> .utils <span class="keyword">import</span> VocabUtility</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">_VocabParallelCrossEntropy</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Maximum value along vocab dimension across all GPUs.</span></span><br><span class="line">        logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">        )</span><br><span class="line">        <span class="comment"># Subtract the maximum value.</span></span><br><span class="line">        vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">        get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">        partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">        rank = get_tensor_model_parallel_rank()</span><br><span class="line">        world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">        vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">        target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">        masked_target = target.clone() - vocab_start_index</span><br><span class="line">        masked_target[target_mask] = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get predicted-logits = logits[target].</span></span><br><span class="line">        <span class="comment"># For Simplicity, we convert logits to a 2-D tensor with size</span></span><br><span class="line">        <span class="comment"># [*, partition-vocab-size] and target to a 1-D tensor of size [*].</span></span><br><span class="line">        logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">        masked_target_1d = masked_target.view(-<span class="number">1</span>)</span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">        predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">        predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br><span class="line">        predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">        predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">        <span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            predicted_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Sum of exponential of logits along vocab dimension across all GPUs.</span></span><br><span class="line">        exp_logits = vocab_parallel_logits</span><br><span class="line">        torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line">        sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            sum_exp_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Loss = log(sum(exp(logits))) - predicted-logit.</span></span><br><span class="line">        loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Normalize and optionally smooth logits</span></span><br><span class="line">        exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        vocab_size = exp_logits.size(-<span class="number">1</span>)</span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            We&#x27;d like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + alpha * mean(y_&#123;i for i != gt&#125;)</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = (K * (1 - alpha) - 1) / (K - 1)) * y_gt  + (alpha / (K - 1)) * \sum_&#123;i&#125; y_i</span></span><br><span class="line"><span class="string">            = (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_&#123;i&#125; y_i / K</span></span><br><span class="line"><span class="string">            From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py</span></span><br><span class="line"><span class="string">            &quot;&quot;&quot;</span></span><br><span class="line">            <span class="keyword">assert</span> <span class="number">1.0</span> &gt; label_smoothing &gt; <span class="number">0.0</span></span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.</span></span><br><span class="line">            log_probs = torch.log(exp_logits)</span><br><span class="line">            mean_log_probs = log_probs.mean(dim=-<span class="number">1</span>)</span><br><span class="line">            loss = (<span class="number">1.0</span> - smoothing) * loss - smoothing * mean_log_probs</span><br><span class="line"></span><br><span class="line">        ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Store softmax, target-mask and masked-target for backward pass.</span></span><br><span class="line">        ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Retreive tensors from the forward path.</span></span><br><span class="line">        softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">        label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># All the inputs have softmax as thier gradient.</span></span><br><span class="line">        grad_input = softmax</span><br><span class="line">        <span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">        partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">        grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">        softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= (<span class="number">1.0</span> - smoothing) * softmax_update</span><br><span class="line">            average_grad = <span class="number">1</span> / vocab_size</span><br><span class="line">            grad_2d[arange_1d, :] -= smoothing * average_grad</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Finally elementwise multiplication with the output gradients.</span></span><br><span class="line">        grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> grad_input, <span class="literal">None</span>, <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">vocab_parallel_cross_entropy</span>(<span class="params">vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">    Performs cross entropy loss when logits are split across tensor parallel ranks</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Arguments:</span></span><br><span class="line"><span class="string">        vocab_parallel_logits: logits split across tensor parallel ranks</span></span><br><span class="line"><span class="string">                               dimension is [sequence_length, batch_size, hidden_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        target: correct vocab ids of dimseion [sequence_length, micro_batch_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        lobal_smoothing: smoothing factor, must be in range [0.0, 1.0)</span></span><br><span class="line"><span class="string">                         default is no smoothing (=0.0)</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">return</span> _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing)</span><br></pre></td></tr></table></figure>
+<h3 id="为什么">为什么</h3>
+<p>本文不介绍分布式训练的其他原理，仅关注模型最后的输出，以及损失函数的计算。</p>
+<p>前文已经详细介绍了交叉熵的实现，那么为什么到分布式训练的时候，就不能直接复用呢？主要原因在于模型的输出被拆散了。回顾模型的输出，其shape为[<code>bs</code>,
+<code>C</code>]，<code>bs</code>表示每次训练的数据量<code>batch size</code>，<code>C</code>表示任务的总的分类数量，在LLM中主要是词表大小<code>vocab_size</code>（下文用此代表类别数量）。</p>
+<p>对于模型的输出而言，分布式（假设有<code>word_size</code>张GPU）主要分两种情况
+1.
+拆分<code>bs</code>，每张卡上有<code>bs</code>数据量，那么整合模型的最后输出应为[<code>bs</code>
+* <code>word_size</code>, <code>vocab_size</code>] 2.
+拆分<code>vocab_size</code>，单个模型的输出为[<code>bs</code>,
+<code>vocab_size</code> //
+<code>word_size</code>]。这种情况需要保证<code>vocab_size</code>能够整除<code>word_size</code></p>
+<p>因此，在进行loss计算前，需要先同步所有的模型输出，然后再进行loss计算。这里的同步，就是从所有GPU中获取模型的输出。但考虑到通信开销问题，有些操作是可以进行优化的，即无需通信全部内容再计算。重新思考损失的计算过程可以分为如下几步：</p>
 <ol type="1">
-<li>等式约束
+<li>SoftMax计算：<span class="math inline">\(\exp(x) /
+\sum(\exp(x)\)</span></li>
+<li>log的计算： <span class="math inline">\(-\log[\exp(x) /
+\sum(\exp(x)]\)</span></li>
+<li>索引：根据targets索引矩阵对应位置 <span class="math inline">\(x[...,
+...]\)</span></li>
+</ol>
+<p>其中，前两者跟最后一个步骤是可以独立运行的。而前两者的计算是耦合的，为节约通信量，对log计算进行变化</p>
 <ol type="1">
-<li>线性等式约束</li>
-<li>非线性等式约束</li>
-</ol></li>
-<li>不等式约束
+<li><span class="math inline">\(-\log[\exp(x) /
+\sum(\exp(x)]\)</span></li>
+<li><span class="math inline">\(-\log[\exp(x)] +
+\log[\sum\exp(x)]\)</span></li>
+<li><span class="math inline">\(\log[\sum\exp(x)] - x\)</span></li>
+</ol>
+<p>因此，可以由原始的<code>func1</code>推导至分布式的<code>func2</code>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func1</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 原始版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    <span class="comment"># Step 0:</span></span><br><span class="line">    predicted_logits = outputs[<span class="built_in">range</span>(bs), targets]</span><br><span class="line"></span><br><span class="line">    predicted_logits = torch.softmax(predicted_logits, dim=<span class="number">1</span>)</span><br><span class="line">    loss = -torch.log(predicted_logits)</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func2</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 分布式的简化版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">    min_c, max_c = <span class="number">0</span>, <span class="number">3</span> <span class="comment"># 预先设定的最小和最大值</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 只选择在该设备上的数值</span></span><br><span class="line">    target_mask = (targets &lt; min_c) | (targets &gt;= max_c)</span><br><span class="line">    <span class="comment"># 由于在分布式中min_c不一定为0，所以需要做一个归一化</span></span><br><span class="line">    masked_target = targets.clone() - min_c</span><br><span class="line">    masked_target[target_mask] = min_c</span><br><span class="line"></span><br><span class="line">    <span class="comment"># arange_1d, masked_target 分别对应 上文的range(bs), targets</span></span><br><span class="line">    arange_1d = torch.arange(start=<span class="number">0</span>, end=bs)</span><br><span class="line">    predicted_logits = outputs[arange_1d, masked_target]</span><br><span class="line">    </span><br><span class="line">    predicted_logits = predicted_logits.clone().contiguous()</span><br><span class="line">    <span class="comment"># 对于非该设备上计算的值，置为0。则不进行计算</span></span><br><span class="line">    predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">    <span class="comment"># 先算第二项</span></span><br><span class="line">    predicted_logits = torch.<span class="built_in">sum</span>(predicted_logits, dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">    loss = torch.log(torch.exp(outputs).<span class="built_in">sum</span>(dim=-<span class="number">1</span>)) - predicted_logits</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])    </span><br><span class="line"></span><br><span class="line">func1(outputs, targets)</span><br><span class="line">func2(outputs, targets)</span><br></pre></td></tr></table></figure></p>
+<h3 id="forward-1">Forward</h3>
+<h4 id="预处理">预处理</h4>
+<p>一般而言，为了数值稳定性，SoftMax计算前会减去最大值，即</p>
+<p><span class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}} = \frac {e^{x_i - \max(x)}}{\sum_{j=1}^n
+e^{x_j - \max(x)}}\]</span></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每张卡上的模型输出为 vocab_parallel_logits，shape：[bs, seq_len, vocab_size // word_size]</span></span><br><span class="line"><span class="comment"># SoftMax的前置计算，可以无需通信全部参数，分两步进行</span></span><br><span class="line"><span class="comment"># 1. 计算每张卡的最大值</span></span><br><span class="line"><span class="comment"># 2. 再计算所有卡的最大值</span></span><br><span class="line"><span class="comment"># 3. 最后对每卡张减去最大值</span></span><br><span class="line"><span class="comment"># 这样就能避免通信所有的参数</span></span><br><span class="line">logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">)</span><br><span class="line"><span class="comment"># Subtract the maximum value.</span></span><br><span class="line">vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>这里的<code>seq_len</code>是LLM中句子的长度，句子的每个位置（token）会计算下一个token的概率（在词表中选），因此，可以理解为每个位置都是一个分类任务，因此<code>vocab_size</code>就是分类的数量。即，可以转换为[<code>bs</code>*<code>seq_len</code>,
+<code>vocab_size</code> // <code>word_size</code>]的形式</p>
+<p>由于单卡不一定存在所有词表的值，因此需要在这张卡上将非该卡输出位置的值置为0。
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">rank = get_tensor_model_parallel_rank()</span><br><span class="line">world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">masked_target = target.clone() - vocab_start_index</span><br><span class="line">masked_target[target_mask] = <span class="number">0</span></span><br></pre></td></tr></table></figure></p>
+<h4 id="根据-targets-选择模型真实输出的logits">根据 targets
+选择模型真实输出的Logits</h4>
+<p>将模型的输出和标签 转换为[<code>bs</code>*<code>seq_len</code>,
+<code>vocab_size</code> // <code>word_size</code>]的形式，以便于计算
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">masked_target_1d = masked_target.view(-<span class="number">1</span>) </span><br><span class="line"></span><br><span class="line"><span class="comment"># 选择真实标签的logits，相当于原来的outputs[range(bs), targets]</span></span><br><span class="line"><span class="comment"># 相当于把此步骤置前，而对于后续计算无影响，并减少了后续的通信时间</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br></pre></td></tr></table></figure></p>
+<h4 id="计算loss">计算loss</h4>
+<p>将logits转换为原来的shape，再求和，先得到了最后的第一项，之后再减去log
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line"><span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    predicted_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure></p>
+<p>计算SoftMax <span class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">exp_logits = vocab_parallel_logits</span><br><span class="line"><span class="comment"># 计算每张卡的exp，作为分子</span></span><br><span class="line">torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line"><span class="comment"># 再计算这张卡上的exp之和</span></span><br><span class="line">sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 再计算所有卡的exp之和，作为分母</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    sum_exp_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure>
+<p>计算loss <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一项是log(sum(exp(logits)))，第二项是最后的logits</span></span><br><span class="line">loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为了Backward，需要保存一些变量</span></span><br><span class="line">exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure></p>
+<h3 id="backward-1">Backward</h3>
+<h4 id="预处理-1">预处理</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据SoftMax计算的结果，忽略无需计算梯度的位置。target_mask中的元素不是0就是1</span></span><br><span class="line">grad_input = softmax</span><br><span class="line"><span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br></pre></td></tr></table></figure>
+<h4 id="梯度计算">梯度计算</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 正常计算位置，就-1，否则将不减（即不做计算）</span></span><br><span class="line">grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后乘上关于loss的梯度</span></span><br><span class="line">grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Algorithm</category>
+      </categories>
+      <tags>
+        <tag>PyTorch</tag>
+        <tag>数学</tag>
+        <tag>交叉熵</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Birthday in Pi：查找日期在Pi中的位置</title>
+    <url>/hexo_blog/2021/03/18/Python/Birthday_in_Pi/</url>
+    <content><![CDATA[<p>想知道自己的生日在圆周率π的第几位出现吗？本文介绍了从思路到实现所遇到的问题，详细分析了如何使用迭代器优化性能，并改用Python语言实现了类似“
+锁”内存等操作。最终，将函数打包成了exe程序，在公众号内提供查询服务，还提供了一个不断更新的生日与位置的查表。</p>
+<span id="more"></span>
+<p>完整的项目代码均开源: <a
+href="https://github.com/wnma3mz/birthday_in_pi">https://github.com/wnma3mz/birthday_in_pi</a></p>
+<h2 id="想法">想法</h2>
+<p>于我而言，很多项目的出发点都是源于浏览东西时所想，觉得有意思就去实现。比如这次是在即刻上看到北大数院公众号，可以回复生日（如20000101）就可以知道这串数字在Pi的第多少位出现。更详细的例子：已知3.1415926...，那么1415就是出现在第1位（从1开始计数，小数点后开始计数）。但该公众号仅在Pi的2亿位数中进行查找，所以有即友回复，搜不到结果。故想到可以试试更长的数据集是否能够搜到。</p>
+<p><strong>背景知识</strong></p>
+<p>理论上，Pi里面可以查找所有人的电话号码、生日、银行卡密码等等。</p>
+<h2 id="行动">行动</h2>
+<h3 id="数据">数据</h3>
+<p>巧妇难为无米之炊，这里最重要的就是Pi这个无理数数据。</p>
 <ol type="1">
-<li>线性等式约束</li>
-<li>非线性等式约束</li>
-</ol></li>
-</ol></li>
+<li>是否有现成的数据文本-&gt;没搜到，搜到的位数都很小，不足以满足需求</li>
+<li>计算Pi（生成超过2亿数的文本）
+<ol type="1">
+<li>自己手写</li>
+<li>找程序。对于计算Pi的项目，本身就是一个科学问题，故一定有现有程序可以辅助完成。所以找到了<a
+href="http://www.numberworld.org/y-cruncher/">y-cruncher</a>项目。具体是否为当前最快的生成程序，未深究，不过实际测试，效率已经可以满足需求。（用i5-1035G1的CPU，生成25亿数，8个线程计算用了10分钟不到，文本大小2.4G）</li>
 </ol></li>
 </ol>
-<h3 id="全局优化">全局优化</h3>
-<ol type="1">
-<li>启发式算法
+<h3 id="编码">编码</h3>
+<p>该问题本身简单来看就是一个字符串查找问题，所以用Python的话，<code>find</code>函数可以直接完成这项任务。但一旦问题变"大"（数据集变大、机器变多......），就是一个"难"题了。</p>
+<p>当然，先不管如何，试试最粗暴的解法看看性能如何。根据测试结果，有两个问题，时间超过5s,
+内存占用2G（文本大小2G）。就算时间可以接受，那么这个空间占用也很难接受。试想，每查询一次都需要占用2G内存，查询完再释放。所以直接想到的优化点有以下几个</p>
 <ol type="1">
-<li>模拟退火</li>
-<li>遗传算法</li>
-</ol></li>
-<li>基本算法
+<li>"锁"内存，即每次查询完不释放这个内存，这样IO时间就可以大大减小</li>
+<li>"做表"，查询的生日数字是有限的，无非就是(19700101-20101231有限数据)，就算100年，也只是36500个数据（实际会略高，因为有闰年）。查表的效率就非常快了</li>
+<li>从根本上优化这个程序</li>
+</ol>
+<h3 id="优化">优化</h3>
+<p>首先排除2，因为最终还是想查找任意数字，2不便于后续扩展，而且根据现有的性能，做表也需要很长时间。</p>
+<p>其次对于1，如果考虑对外开放，服务器占用2G内存空间，实在是太"奢侈"了</p>
+<p>所以最后就只能考虑3。</p>
+<h4 id="迭代器">迭代器</h4>
+<p>Python有个关键词是
+<code>yield</code>，它可以实现迭代器这个功能。说人话就是，它不是一次性把数据加载出来，而且每次只加载一部分。下面是一个简单的例子</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">yield_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">for_return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">return</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">return</span> lst</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    lst = <span class="built_in">list</span>(<span class="built_in">range</span>(<span class="number">10</span>))</span><br><span class="line">    <span class="built_in">print</span>(yield_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(for_return_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(return_func(lst))</span><br><span class="line">    <span class="keyword">for</span> x <span class="keyword">in</span> yield_func(lst):</span><br><span class="line">        <span class="built_in">print</span>(x, end=<span class="string">&quot; &quot;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line">&lt;generator object yield_func at 0x000001644AF624F8&gt;</span><br><span class="line">0</span><br><span class="line">[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span><br><span class="line">0 1 2 3 4 5 6 7 8 9</span><br></pre></td></tr></table></figure>
+<p>所以肯定是要利用这个方法来帮助优化程序。在Python操作文件的时候
+<code>readline()</code>函数就是一个生成器，一行一行的返回，查找速度比直接
+<code>read()</code>或者 <code>readlines()</code>更快。</p>
+<p>但这里的问题是，输出的Pi文本是一行。。。所以没法直接使用。当然也可以手动把这个文本按行切割后，再用
+<code>readline()</code>函数。</p>
+<p>这里我找到的解决方案[1]可以更方便的解决这个问题，详见博客or项目源码。</p>
+<h4 id="bug">bug</h4>
+<p>把程序写好之后，可以发现一个很明显的bug，就是查询的字符串如果在两个
+<code>chunk</code>之间，是会直接跳过。比如
+<code>12345678</code>，<code>block_size=4</code>，那么就是
+<code>1234</code>和 <code>5678</code>。如果查找
+<code>45</code>是找不到的。</p>
+<p>所以这里需要做两个额外的处理</p>
 <ol type="1">
-<li>随机搜索</li>
-<li>多起点优化</li>
-</ol></li>
+<li>加上上一个 <code>block_size</code>的字符串
+<code>old_chunk = copy.deepcopy(chunk[-len(target_str) :])</code>。这里为了省空间和时间，只加上了需要查询字符串的长度，就可以满足需求</li>
+<li>返回查找位数时需要额外
+<code>-len(old_chunk)</code>。（注：这里一开始忘了操作，故前几位查询的既有会发现结果与北大数院查询的结果差10位，在这里致歉）</li>
 </ol>
-<h3 id="优化问题fx">优化问题<span
-class="math inline">\(F(x)\)</span></h3>
-<ul>
-<li>线性问题
-<ul>
-<li>不论有无约束，都可以使用<strong>单纯形法</strong>和<strong>内点法</strong>求解</li>
-</ul></li>
-<li>非线性问题
-<ul>
-<li>凸优化问题
-<ul>
-<li>有约束：切平面法、椭球法、内点法</li>
-<li>无约束情况下等同于非凸问题</li>
-</ul></li>
-<li>非凸问题
-<ul>
-<li>约束<span
-class="math inline">\(h(x)\)</span>不是线性，拉格朗日乘子法</li>
-<li>约束<span class="math inline">\(g(x)\)</span>为线性，gradient
-projection、SQP</li>
-<li>约束<span
-class="math inline">\(g(x)\)</span>不是线性，惩罚函数、Barrier函数、SQP问题</li>
-<li>无约束（可消约束）
-<ul>
-<li>多起点优化
-<ul>
-<li>Hession矩阵，一阶矩阵：Levenberg-Marquardt、牛顿法、牛顿线性搜索</li>
-<li>一阶矩阵：Quasi-Newton、Quasi-Newton线性搜索、最速下降法</li>
-<li>无：Powell Perpendicular线性搜索、Nelder-Mead</li>
-</ul></li>
-<li>不是多起点优化：随机搜索、模拟退火、遗传算法</li>
-</ul></li>
-</ul></li>
-</ul></li>
-<li>二次问题
-<ul>
-<li>约束，线性，修正的单纯形法、内点法</li>
-</ul></li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E7%9A%84%E9%80%89%E6%8B%A9.png" />
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E6%9C%80%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E5%81%9C%E6%AD%A2%E5%87%86%E5%88%99.png" /></p>
-<h3 id="例子">例子</h3>
-<p>非凸函数</p>
-<p><span class="math inline">\(\int _ { 0 } ^ { x 2 } ( \frac { e ^ { x
-_ { 3 } ^ { 2 } z } \cos ( z ) } { x _ { 3 } ^ { 4 } z + 1 } + z ^ { x 1
-} ) d z\)</span></p>
-<p><span class="math inline">\(e ^ { x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2
-} } + x _ { 1 } ^ { 2 } x _ { 2 } ^ { 2 }\)</span></p>
-<p><span class="math inline">\(( x _ { 1 } + x _ { 2 } ) ^ { 3 } - \log
-( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )\)</span></p>
-<p>$x _ { 1 } ^ { 4 } + x _ { 2 } ^ { 2 } + 3 x _ { 1 } x _ { 3 } + 2 x
-_ { 4 } x _ { 5 } $</p>
-<p>$9 x _ { 1 } ^ { 2 } + 4 x _ { 2 } ^ { 2 } + 2 x _ { 1 } x _ { 2 } +
-8 x _ { 3 } ^ { 2 } - 2 x _ { 1 } x _ { 3 } $</p>
-<p>$ { ( x _ { 1 } + 3 x _ { 2 } + 4 x _ { 3 } ) ^ { 2 } } + ( ( x _ { 1
-} + 4 x _ { 2 } + 5 x _ { 3 } - 100 ) ^ { 4 } )$</p>
-<p>$ { 1 + } $</p>
-<p>凸函数</p>
-<p>$( x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2 } ) + 14 x _ { 2 } ^ { 4 }
-$</p>
-<p>$ ( ( x _ { 1 } - 1 ) ^ { 2 } + ( x _ { 2 } - 2 ) ^ { 2 } ) ^ { 2 } -
-( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )$</p>
-<p>$| 2 x _ { 1 } + 4 x _ { 2 } + 3 x _ { 3 } + 2 x _ { 4 } + 7 | + ( (
-x _ { 1 } + x _ { 2 } - 4 x _ { 3 } - x _ { 4 } ) ^ { 4 } ) $</p>
-<p>$( F _ { 1 } ( x ) , 2 F _ { 2 } ( x ) , F _ { 3 } ( x ) + 8 F _ { 4
-} ( x ) ) $</p>
-<p>$ x _ { 1 } + | x _ { 2 } | + 3 x _ { 3 } + 4 | x _ { 4 } |$</p>
-<p><span class="math inline">\(( 4 x _ { 1 } + 7 x _ { 2 } - 4 x _ { 3 }
-+ 12 x _ { 4 } ) ^ { 3 }\)</span></p>
-<p><span class="math inline">\(( x _ { 1 } - 5 ) ^ { 2 } + ( x _ { 2 } -
-8 ) ^ { 4 }\)</span></p>
-<p>线性规划问题</p>
-<p><span class="math inline">\(| x _ { 1 } - 4 x _ { 2 } + x _ { 3 } - 4
-|\)</span></p>
-<p>$( 4 x _ { 1 } - 3 x _ { 2 } + 8 x _ { 3 } - 5 , - 2 x _ { 1 } + 7 x
-_ { 2 } - x _ { 3 } + 1 ) $</p>
-<p>二次规划（凸二次函数）</p>
-<p><span class="math inline">\(2 ^ { 5 x _ { 1 } ^ { 2 } + 2 x _ { 2 } ^
-{ 2 } + 6 x _ { 1 } x _ { 2 } + 3 x _ { 1 } - x _ { 2 } }\)</span></p>
-<p><span class="math inline">\(1 - 4 x _ { 1 } ^ { 2 } - 9 x _ { 2 } ^ {
-2 } - 8 x _ { 3 } ^ { 2 } + 4 x _ { 1 } x _ { 2 } - 2 x _ { 1 } x _ { 3
-} + 4 x _ { 2 } x _ { 3 }\)</span></p>
-<h3 id="相关书籍">相关书籍</h3>
-<p>[1] 程理民, 吴江, 张玉林. 运筹学模型与方法教程[M]. 清华大学出版社,
-2000.</p>
-<p>[2] 陈宝林. 最优化理论与算法[M]. 清华大学出版社有限公司, 2005.</p>
-<p>[3] 袁亚湘, 孙文瑜. 最优化理论与方法[M]. 科学出版社, 1997.</p>
-<p>个人感觉，难度[3]&gt;[2]&gt;[1]，袁院士那本理论证明偏多，可搭配[2]与课后练习一起阅读。</p>
+<p>最后经过测试后，性能基本达到预期（见项目README，有说明具体结果）。</p>
+<h2 id="部署">部署</h2>
+<p>至此，核心程序已经完成，接下来需要考虑一种方式来给其他人方便使用了。最直接的其实就是写一个Web，前端去调用后端返回结果。但同样的，这里还是考虑到网站流量等原因，跟北大数院一样放到公众号上也很方便，开发上也只需要完成后端即可，无需考虑前端UI啥的。</p>
+<p>查询公众号文档后，一通操作，就把功能完成了（文案直接"抄"了北大数院的文案）。代码见<a
+href="https://github.com/wnma3mz/birthday_in_pi/blob/master/flask_server.py">flask_server.py</a></p>
+<p>这里需要在微信公众号后台配置开发者模式，然后把token复制进代码。</p>
+<h3 id="tip">Tip</h3>
+<p>公众号开发要求是80端口，但这里通过子域名的方式+nginx反代就可以直接用域名访问，而无需80端口。nginx配置见下</p>
+<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">listen</span>       <span class="number">80</span>;</span><br><span class="line">    <span class="attribute">server_name</span>  xxx.yyy.zzz;</span><br><span class="line"></span><br><span class="line">    <span class="section">location</span> / &#123;</span><br><span class="line">        <span class="attribute">uwsgi_pass</span> <span class="number">0.0.0.0</span>:port;</span><br><span class="line">        <span class="attribute">proxy_pass</span> http://0.0.0.0:port/wx_api;</span><br><span class="line">    &#125;</span><br><span class="line">  </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>这里的 <code>xxx.yyy.zzz</code>就是域名，port就是
+<code>flask</code>开启的端口</p>
+<h2 id="其他">其他</h2>
+<h3 id="查表">查表</h3>
+<p>当然，查表的方式还是最快的，所以还是建了一个表，并且做了一个简单的分析。分析内容如下</p>
+<p>最近只需246位能够找到,
+最远需要近9亿位。列出Top10（左侧为年月日，右侧为第x位）</p>
+<p>日期范围为19700101-20091231</p>
+<table>
+<thead>
+<tr>
+<th>日期</th>
+<th>位数</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>20190914</td>
+<td>246</td>
+</tr>
+<tr>
+<td>20190516</td>
+<td>5364</td>
+</tr>
+<tr>
+<td>19910403</td>
+<td>8365</td>
+</tr>
+<tr>
+<td>20141020</td>
+<td>9813</td>
+</tr>
+<tr>
+<td>19950728</td>
+<td>1,3874</td>
+</tr>
+<tr>
+<td>20121031</td>
+<td>1,4538</td>
+</tr>
+<tr>
+<td>19990814</td>
+<td>2,5593</td>
+</tr>
+<tr>
+<td>19810705</td>
+<td>2,7202</td>
+</tr>
+<tr>
+<td>19921122</td>
+<td>3,0884</td>
+</tr>
+<tr>
+<td>19930509</td>
+<td>3,6615</td>
+</tr>
+<tr>
+<td>..........</td>
+<td>..........</td>
+</tr>
+<tr>
+<td>19941227</td>
+<td>7,1993,2960</td>
+</tr>
+<tr>
+<td>20080923</td>
+<td>7,2930,2729</td>
+</tr>
+<tr>
+<td>19911201</td>
+<td>7,2985,0775</td>
+</tr>
+<tr>
+<td>20071206</td>
+<td>7,3782,1997</td>
+</tr>
+<tr>
+<td>20090220</td>
+<td>7,5402,4215</td>
+</tr>
+<tr>
+<td>19730829</td>
+<td>7,7357,0338</td>
+</tr>
+<tr>
+<td>20070526</td>
+<td>7,8655,8038</td>
+</tr>
+<tr>
+<td>19951122</td>
+<td>8,2582,0139</td>
+</tr>
+<tr>
+<td>20140407</td>
+<td>8,7320,8091</td>
+</tr>
+<tr>
+<td>19730521</td>
+<td>8,8950,7619</td>
+</tr>
+</tbody>
+</table>
+<p>为什么4位分隔，而不是3位？</p>
+<p>因为我觉得这样看的更舒服，读万、亿更方便。。。</p>
+<h3 id="程序打包">程序打包</h3>
+<p>为方便没有Python程序的用户，同时也不想使用公众号。在Windows上打包了一份exe程序。命令也很简单
+<code>pyinstaller -F .\read_large_pi.py</code>。</p>
+<p>最后的最后，该程序尚存在优化空间，当查询数字较长（超过8位）时，时间花费也更久（超过1s）。</p>
+<h2 id="参考文章">参考文章</h2>
+<p>[1] <a
+href="https://lovemma.github.io/posts/python-%E5%A6%82%E4%BD%95%E6%B5%81%E5%BC%8F%E8%AF%BB%E5%8F%96%E5%A4%A7%E6%96%87%E4%BB%B6/">python-如何流式读取大文件</a></p>
 ]]></content>
       <categories>
-        <category>Algorithm</category>
+        <category>Note</category>
       </categories>
       <tags>
-        <tag>Optimization</tag>
+        <tag>Python</tag>
+        <tag>微信开发</tag>
+        <tag>Steaming</tag>
       </tags>
   </entry>
   <entry>
-    <title>矩阵分解介绍及Python实现（QR、SVD、LU）</title>
-    <url>/hexo_blog/2018/04/13/%E6%95%B0%E5%AD%A6/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3%E4%BB%8B%E7%BB%8D%E5%8F%8APython%E5%AE%9E%E7%8E%B0%EF%BC%88QR%E3%80%81SVD%E3%80%81LU%EF%BC%89/</url>
-    <content><![CDATA[<p>本篇文章简单介绍了三种矩阵分解方式并且附上部分分解方式的实现代码。</p>
+    <title>Python3的Tools-scripts整理</title>
+    <url>/hexo_blog/2020/06/14/Python/Python3%E7%9A%84Tools-scripts%E6%95%B4%E7%90%86/</url>
+    <content><![CDATA[<p>Python3的Tools-scripts整理</p>
 <span id="more"></span>
-<h1 id="矩阵分解">矩阵分解</h1>
-<p>Python实现了QR与LU分解，QR用了两种变换方式实现。<a
-href="https://github.com/wnma3mz/Matrix-factorization">Github地址</a></p>
-<h2 id="what">What？</h2>
+<p>文件路径为<code>Anaconda3\Tools\scripts</code></p>
+<h3 id="byext.py">byext.py</h3>
+<p>分析文件/文件夹</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># py文件</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line">.py   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">TOTAL   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line"></span><br><span class="line"><span class="comment"># 二进制文件</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line">&lt;none&gt;      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line"> TOTAL      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件夹</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br><span class="line"> .png      <span class="number">2</span>   <span class="number">2823</span>             <span class="number">2</span></span><br><span class="line">  .py         <span class="number">10471</span>             <span class="number">5</span>    <span class="number">224</span>    <span class="number">647</span></span><br><span class="line"> .txt         <span class="number">17404</span>             <span class="number">3</span>    <span class="number">554</span>    <span class="number">960</span></span><br><span class="line">&lt;<span class="built_in">dir</span>&gt;                    <span class="number">1</span></span><br><span class="line">TOTAL      <span class="number">2</span>  <span class="number">30698</span>      <span class="number">1</span>     <span class="number">10</span>    <span class="number">778</span>   <span class="number">1607</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br></pre></td></tr></table></figure>
+<table>
+<thead>
+<tr>
+<th>输出</th>
+<th>主要语法</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>ext</td>
+<td><code>os.path.normcase(os.path.splitext[1])</code></td>
+</tr>
+<tr>
+<td>files</td>
+<td><code>os.path.isfile</code></td>
+</tr>
+<tr>
+<td>bytes</td>
+<td>以二进制(<code>rb</code>)读文件，<code>len</code>的输出</td>
+</tr>
+<tr>
+<td>lines</td>
+<td><code>str(data, 'latin-1').splitlines()</code></td>
+</tr>
+<tr>
+<td>words</td>
+<td><code>len(data.split())</code>的输出</td>
+</tr>
+<tr>
+<td>binary</td>
+<td><code>if *b*'\0' in data</code></td>
+</tr>
+<tr>
+<td>dirs</td>
+<td>os.path.isdir</td>
+</tr>
+<tr>
+<td>lnk</td>
+<td>os.path.islink</td>
+</tr>
+</tbody>
+</table>
+<h3 id="checkpip.py">checkpip.py</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">The latest version of setuptools on PyPI <span class="keyword">is</span> <span class="number">46.2</span><span class="number">.0</span>, but ensurepip has <span class="number">40.8</span><span class="number">.0</span></span><br><span class="line">The latest version of pip on PyPI <span class="keyword">is</span> <span class="number">20.1</span>, but ensurepip has <span class="number">19.0</span><span class="number">.3</span></span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 主要是检查版本</span></span><br><span class="line"><span class="keyword">import</span> ensurepip</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(ensurepip._PROJECTS)</span><br><span class="line"><span class="comment"># [(&#x27;setuptools&#x27;, &#x27;40.8.0&#x27;), (&#x27;pip&#x27;, &#x27;19.0.3&#x27;)]</span></span><br></pre></td></tr></table></figure>
+<h3 id="checkpyc.py">checkpyc.py</h3>
+<p>检查pyc文件是否最新</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> importlib.util</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(importlib.util.MAGIC_NUMBER)</span><br><span class="line"><span class="comment"># b&#x27;B\r\r\n&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="cleanfuture.py">cleanfuture.py</h3>
+<p>修改py文件的future语法</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">errprint</span>(<span class="params">*args</span>):</span><br><span class="line">    strings = <span class="built_in">map</span>(<span class="built_in">str</span>, args)</span><br><span class="line">    msg = <span class="string">&#x27; &#x27;</span>.join(strings)</span><br><span class="line">    <span class="keyword">if</span> msg[-<span class="number">1</span>:] != <span class="string">&#x27;\n&#x27;</span>:</span><br><span class="line">        msg += <span class="string">&#x27;\n&#x27;</span></span><br><span class="line">    sys.stderr.write(msg)</span><br><span class="line">    </span><br><span class="line">errprint(<span class="string">&quot;Usage:&quot;</span>, __doc__)</span><br></pre></td></tr></table></figure>
+<h3 id="crlf.py">crlf.py</h3>
+<p>文件中CRLF用LF替换</p>
+<h3 id="lfcr.py">lfcr.py</h3>
+<p>文件中LF用CRLF替换</p>
+<p>迭代：<code>itertools.count()</code></p>
+<h3 id="reindent.py">reindent.py</h3>
+<p>py文件改为4空格缩进</p>
+<h3 id="untabify.py">untabify.py</h3>
+<p>同上</p>
+<h3 id="texi2html.py">texi2html.py</h3>
+<p>texi to html</p>
+<p><code>import keyword</code></p>
+<p><code>import webbrowser</code></p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Scripts</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python杂谈</title>
+    <url>/hexo_blog/2018/03/14/Python/Python%E6%9D%82%E8%B0%88/</url>
+    <content><![CDATA[<p>理解Python中的GIL(Global Interpreter
+Lock)，以及如何使用多线程和多进程，以及弱引用。</p>
+<span id="more"></span>
+<h2 id="gil">GIL</h2>
+<p>建议阅读文章</p>
 <p><a
-href="https://zh.wikipedia.org/wiki/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3">wiki链接</a></p>
-<p>将矩阵拆分成数个三角形矩阵。</p>
-<h2 id="why">Why？</h2>
-<ol type="1">
-<li>数值分析中，用于实现一些矩阵运算的快速算法</li>
-<li>推荐算法（SVD），信号处理（SVD）</li>
-<li>反应矩阵中的一些数值特性：如矩阵的秩、特征值(奇异值)</li>
-</ol>
-<p>每一种矩阵分解方式对应不同的意义</p>
-<h2 id="how">How？</h2>
-<h3 id="基本概念介绍">基本概念介绍</h3>
-<p>关于转置、逆矩阵就不多介绍了。</p>
-<ul>
-<li>满秩矩阵
-<ul>
-<li>矩阵的秩等于行数—&gt;行满秩</li>
-<li>矩阵的秩等于列数—&gt;列满秩</li>
-<li>以上两个都满足称为n阶方阵</li>
-</ul></li>
-<li>酉矩阵（正交矩阵）
+href="http://zhuoqiang.me/python-thread-gil-and-ctypes.html">python
+线程，GIL 和 ctypes</a></p>
+<p><a
+href="https://zhuanlan.zhihu.com/p/20953544">谈谈python的GIL、多线程、多进程</a></p>
+<p>GIL, <code>Global Interpreter Lock</code>，全局解释器锁</p>
+<p>产生原因: 防止多线程并发执行机器码的Mutex。</p>
+<p>好处:
+不需要考虑线程安全问题，在单线程的情况下更快(相对而言)，和C库结合时更加方便</p>
+<p>形象的比方:</p>
+<blockquote>
+<p>我们把整个进程空间看做一个车间，把线程看成是多条不相交的流水线，把线程控制流中的字节码看作是流水线上待处理的物品。Python
+解释器是工人，整个车间仅此一名。操作系统是一只上帝之手，会随时把工人从一条流水线调到另一条——这种“随时”是不由分说的，即不管处理完当前物品与否。</p>
+</blockquote>
+<blockquote>
+<p>若没有 GIL。假设工人正在流水线 A 处理 A1 物品，根据 A1
+的需要将房间温度（一个全局对象）调到了 20
+度。这时上帝之手发动了，工人被调到流水线 B 处理 B1 物品，根据 B1
+的需要又将房间温度调到了 50 度。这时上帝之手又发动了，工人又调回 A
+继续处理 A1。但此时 A1 暴露在了 50 度的环境中，安全问题就此产生了。</p>
+</blockquote>
+<blockquote>
+<p>而 GIL 相当于一条锁链，一旦工人开始处理某条流水线上的物品，GIL
+便会将工人和该流水线锁在一起。而被锁住的工人只会处理该流水线上的物品。就算突然被调到另一条流水线，他也不会干活，而是干等至重新调回原来的流水线。这样每个物品在被处理的过程中便总是能保证全局环境不会突变。</p>
+</blockquote>
+<p>在解释器解释python代码时，首先会获得这把锁，需要I/O操作就是释放这把锁。如果是纯计算程序，没有I/O操作，解释器会每隔100次操作释放一次锁（Python3.2之后使用固定时间(5ms)来释放）。</p>
+<p>GIL只会影响严重依赖CPU的程序（计算型）,
+所以运行Python的时候，同一时间只能执行一个线程(单核CPU下的多线程都只是并发)。运行Python程序就只能占用一个核的CPU资源，哪怕使用了
+<code>threading</code>多线程也是一样。</p>
+<p>CPU密集性代码(循环)：不适用多线程。不断触发GIL，线程来回切换，消耗资源</p>
+<p>IO密集型代码(文件处理，网路爬虫)：多线程能够提高效率。在一个线程等待的同时，切换到另一个线程，不浪费CPU资源从而提升效率。</p>
+<p>多核多线程比单核多线程更差，单核下多线程每次释放GIL，唤醒的线程都能获取GIL锁，可以无缝执行。但是多核下，一个CPU释放完GIL之后，所有CPU的线程又开始竞争，可能会被原来的CPU获取，导致其他CPU线程唤醒之后必须等待到下一次释放。这样造成的结果称为线程颠簸(thrashing),
+效率会更低下。</p>
+<p>以上，如果Python需要利用多核CPU，就使用多进程(<code>multiprocessing</code>)，各自拥有独立的GIL，互不干扰，从而达到并行执行。还有一种方案就是使用C进行拓展，手动释放GIL，这里不做介绍。</p>
+<p>参考代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-   </span></span><br><span class="line"><span class="keyword">from</span> time <span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread</span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Process</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_threads</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Thread(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_processes</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Process(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">test</span>(<span class="params">target, number=<span class="number">10</span>, spawner=spawn_n_threads</span>):</span><br><span class="line">    <span class="keyword">for</span> n <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5</span>):</span><br><span class="line">        start_time = time()</span><br><span class="line">        <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(number):</span><br><span class="line">            spawner(n, target)</span><br><span class="line">        end_time = time()</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Time elapsed with &#123;&#125; branch(es): &#123;:.6f&#125; sec(s)&quot;</span>.<span class="built_in">format</span>(n, end_time - start_time))</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fib</span>():</span><br><span class="line">    a = b = <span class="number">1</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100000</span>):</span><br><span class="line">        a, b = b, a + b</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;threads:&quot;</span>)</span><br><span class="line">    test(fib)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;processes:&quot;</span>)</span><br><span class="line">    test(fib, spawner=spawn_n_processes)</span><br></pre></td></tr></table></figure>
+<p>P.S.</p>
 <ul>
-<li>方块矩阵，元素皆为实数，行与列皆正交的单位向量</li>
-<li>特性：矩阵的转置矩阵为逆矩阵</li>
-<li>定理（det表示行列式）：</li>
-</ul></li>
+<li>并行：同时处理两个及以上的事件在同一时刻发生</li>
+<li>并发：同时处理两个及以上的事件在同一事件间隔内发生</li>
 </ul>
-<p><span class="math display">\[Q^T=Q^{-1}\Leftrightarrow Q^TQ=QQ^T =
-I\]</span></p>
-<p><span
-class="math display">\[1=\det(I)=\det(Q^TQ)=\det(Q^T)\det(Q)=(\det(Q))^2\Rightarrow|Q|
-\pm 1\]</span></p>
-<ul>
-<li><p>单位矩阵</p>
+<h2 id="弱引用">弱引用</h2>
+<p>推荐阅读</p>
+<p><a
+href="http://blog.soliloquize.org/2016/01/21/Python%E5%BC%B1%E5%BC%95%E7%94%A8%E7%9A%84%E4%BD%BF%E7%94%A8%E4%B8%8E%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9/">Python弱引用的使用与注意事项</a></p>
+<p>弱引用与强引用相对，是指不能确保其引用的对象不会被垃圾回收器回收的引用。若一个对象若只被弱引用所引用，则被认为是不可访问（或弱可访问）的，病因此可能在任何时刻被回收。主要作用就是减少循环引用，减少内存中不必要的对象存在的数量。</p>
+<p>Python用
+<code>weakref</code>模块创建对象的弱引用，当引用计数为0或只存在对象的弱引用时将回收这个对象。</p>
+<p>内存管理时使用，cache编程，垃圾回收(GC)</p>
+<h2 id="while-1与while-true">while 1与while True</h2>
+<p><a
+href="http://www.pythoner.com/356.html">Python天坑系列（一）：while
+1比while True更快？</a></p>
+<p>在python2中while 1会比while True更快</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_one</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_true</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    w1_time, wt_time = <span class="number">0</span>, <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        w1 = timeit.timeit(while_one, <span class="string">&quot;from __main__ import while_one&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        w1_time += w1</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        wt = timeit.timeit(while_true, <span class="string">&quot;from __main__ import while_true&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        wt_time += wt</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;while one: %s\nwhile_true: %s&quot;</span> % (w1_time, wt_time))</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 9.27971386909</span><br><span class="line">while_true: 13.5762348175</span><br><span class="line"><span class="comment"># python3的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 14.19957690299998</span><br><span class="line">while_true: 13.785004297995329</span><br></pre></td></tr></table></figure>
+<p>原因：python2中True/False不是关键字，python3添加了True/False做为关键字。每次在循环的时候，True/False需要被检查，所以while
+1会更快</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Python <span class="number">2.7</span><span class="number">.14</span> (default, Sep <span class="number">23</span> <span class="number">2017</span>, <span class="number">22</span>:06:<span class="number">14</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux2</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;exec&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;print&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br><span class="line"></span><br><span class="line">Python <span class="number">3.6</span><span class="number">.3</span> (default, Oct  <span class="number">3</span> <span class="number">2017</span>, <span class="number">21</span>:<span class="number">45</span>:<span class="number">48</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;False&#x27;</span>, <span class="string">&#x27;None&#x27;</span>, <span class="string">&#x27;True&#x27;</span>, <span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;nonlocal&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br></pre></td></tr></table></figure>
+<h2 id="if-x-true-与-if-x">if x == True 与 if x</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x_eq_true</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x == <span class="literal">True</span>:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    if1 = timeit.timeit(if_x_eq_true, <span class="string">&quot;from __main__ import if_x_eq_true&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line">    if2 = timeit.timeit(if_x, <span class="string">&quot;from __main__ import if_x&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;if_x_eq_true: %s\nif_x: %s&quot;</span> % (if1, if2))</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line">if_x_eq_true: 0.118808984756</span><br><span class="line">if_x: 0.0920550823212</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line">if_x_eq_true: 0.0951261969985353</span><br><span class="line">if_x: 0.07899275699855934</span><br></pre></td></tr></table></figure>
+<p>不管python2还是python3, <code>if x:</code>从各方面来说都会优于
+<code>if x == True</code>。后者会比前者多出一个检查True值和比较的操作。</p>
+<h2 id="type与object">type与object</h2>
+<p><a href="https://www.zhihu.com/question/38791962">Python 的 type 和
+object 之间是怎么一种关系？</a></p>
+<h2
+id="深刻理解python中的元类metaclass">深刻理解Python中的元类(metaclass)</h2>
+<p><a
+href="http://blog.jobbole.com/21351/">深刻理解Python中的元类(metaclass)</a></p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>GIL</tag>
+        <tag>元类</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python 自动导入包</title>
+    <url>/hexo_blog/2023/12/03/Python/Python%E8%87%AA%E5%8A%A8%E5%AF%BC%E5%85%A5%E5%8C%85/</url>
+    <content><![CDATA[<p>在一个文件夹中，有很多个 Python
+文件，每个文件都有若干重名函数，需要一一导入。这个时候，可以用 Python
+的自动导入包来实现。</p>
+<span id="more"></span>
+<h2 id="需求描述">需求描述</h2>
+<p>假设有一个文件夹</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">func/</span><br><span class="line">├── __init__.py</span><br><span class="line">├── func1.py</span><br><span class="line">├── func2.py</span><br><span class="line">├── func3.py</span><br></pre></td></tr></table></figure>
+<p>每个文件都有一个重名函数，比如</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># func1.py</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">foo</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;foo&quot;</span>)</span><br></pre></td></tr></table></figure>
+<p>这不仅导致 <code>import</code> 代码不够简洁，且在新增文件时，还得在
+<code>__init__.py</code> 中添加 <code>import</code>
+语句。甚至在调用处也得增加代码来处理。</p>
+<h2 id="解决方案">解决方案</h2>
+<p>Python 的自动导入包可以解决这个问题。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> glob</span><br><span class="line"><span class="keyword">import</span> importlib</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取当前文件夹下所有的 .py 文件</span></span><br><span class="line">files = glob.glob(os.path.join(os.path.dirname(__file__), <span class="string">&quot;*.py&quot;</span>))</span><br><span class="line">func_dict = &#123;&#125;</span><br><span class="line"><span class="keyword">for</span> file_ <span class="keyword">in</span> files:</span><br><span class="line">    func_name = os.path.basename(file_)[:-<span class="number">3</span>]</span><br><span class="line">    load_module = importlib.import_module(<span class="string">f&quot;func.<span class="subst">&#123;func_name&#125;</span>&quot;</span>)</span><br><span class="line">    <span class="built_in">globals</span>()[load_module.__name__] = <span class="built_in">getattr</span>(load_module, <span class="string">&quot;foo&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将其放到一个字典中，以按需处理，</span></span><br><span class="line">    func_dict[func_name] = <span class="built_in">globals</span>()[load_module.__name__]</span><br></pre></td></tr></table></figure>
+<p>这样，不仅不需要在 <code>func/__init__.py</code> 中添加
+<code>import</code>
+语句，也不需要在调用处添加额外的代码。甚至，还要根据需求，从字典中调用对应函数。当然，这需要提前进行约定，比如全部放到<code>func</code>文件夹下，且都用<code>foo</code>这个名字作为函数名。</p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>importlib</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python魔术方法</title>
+    <url>/hexo_blog/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Python的魔术方法</p>
+<span id="more"></span>
+<p>参考文章:</p>
+<p><a
+href="http://pycoders-weekly-chinese.readthedocs.io/en/latest/issue6/a-guide-to-pythons-magic-methods.html">Python
+魔术方法指南</a></p>
+<p>魔术方法，顾名思义是一种可以给对象(类)增加魔法的特殊方法，它们的表示方法一般是用双下划线包围(如<code>__init__</code>)</p>
+<h2 id="对象实例化">对象实例化</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> os.path <span class="keyword">import</span> join</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">FileObject</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;给文件对象进行包装从而确认在删除时文件流关闭&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, filepath=<span class="string">&#x27;~&#x27;</span>, filename=<span class="string">&#x27;sample.txt&#x27;</span></span>):</span><br><span class="line">        <span class="comment">#读写模式打开一个文件</span></span><br><span class="line">        <span class="variable language_">self</span>.file = <span class="built_in">open</span>(join(filepath, filename), <span class="string">&#x27;r+&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__del__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.file.close()</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.file</span><br></pre></td></tr></table></figure>
 <ul>
-<li>矩阵中的每个元素都为1</li>
-<li>如下</li>
+<li><p><code>__new__(cls, [...])</code>:
+在一个对象实例化调用的第一个方法，第一个参数是类，剩下的参数传递给<code>__init__</code></p></li>
+<li><p><code>__init__(self, [...])</code>:
+类的初始化方法。当构造函数被调用的时候的任何参数够会传给它</p></li>
+<li><p><code>__del__(self)</code>:
+<code>__new__</code>和<code>__init__</code>为对象的构造器，<code>__del__</code>为对象的析构器。当一个对象进行过垃圾回收时候的行为就是这个方法。当一个对象在删除的时候需要更多的清洁工作，就需要使用这个方法</p></li>
 </ul>
-<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 1
-&amp; 1 \\ 1 &amp; 1 &amp; 1 \\ 1&amp; 1 &amp; 1 \end{array}
-\right|\]</span></p></li>
-<li><p>上三角矩阵</p>
+<h2 id="比较">比较</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Word</span>(<span class="title class_ inherited__">str</span>):</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;存储单词的类，定义比较单词的几种方法&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__new__</span>(<span class="params">cls, word</span>):</span><br><span class="line">        <span class="comment"># 注意我们必须要用到__new__方法，因为str是不可变类型</span></span><br><span class="line">        <span class="comment"># 所以我们必须在创建的时候将它初始化</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">&#x27; &#x27;</span> <span class="keyword">in</span> word:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;Value contains spaces. Truncating to first space.&quot;</span>)</span><br><span class="line">            word = word[:word.index(<span class="string">&#x27; &#x27;</span>)] <span class="comment">#单词是第一个空格之前的所有字符</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">str</span>.__new__(cls, word)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__gt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__lt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__ge__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt;= <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__le__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt;= <span class="built_in">len</span>(other)</span><br></pre></td></tr></table></figure>
 <ul>
-<li>非零元素在右上方的矩阵</li>
-<li>如下</li>
+<li><p><code>__cmp__(self, other)</code>:
+用于比较的魔术方法，实现了所有的比较符号。如果<code>self</code> &lt;
+<code>other</code>返回一个负数；如果<code>self</code> ==
+<code>other</code>返回０；如果<code>self</code> &gt;
+<code>other</code>返回正数。一般情况下是分别定义每一个比较符号。如果想实现所有的比较符号，就需要调用这个方法。</p></li>
+<li><p><code>__eq__(self, other)</code>: 等号</p></li>
+<li><p><code>__ne__(self, other)</code>:　不等号</p></li>
+<li><p><code>__lt__(self, other)</code>:　小于</p></li>
+<li><p><code>__gt__(self, other)</code>:　大于</p></li>
+<li><p><code>__le__(self, other)</code>:　小于等于</p></li>
+<li><p><code>__ge__(self, other)</code>:　大于等于</p></li>
 </ul>
-<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 2
-&amp; 3 \\ 0 &amp; 1 &amp; 2\\ 0&amp; 0 &amp; 1 \end{array}
-\right|\]</span></p></li>
-<li><p>二范数</p>
+<h2 id="一元操作符和函数">一元操作符和函数</h2>
 <ul>
-<li>假设有一个x的向量</li>
+<li><code>__pos__(self)</code>：实现正号的特性</li>
+<li><code>__neg__(self)</code>：实现负号的特性</li>
+<li><code>__abs__(self)</code>：abs, 绝对值</li>
+<li><code>__invert__(self)</code>：实现~的特性</li>
 </ul>
-<p><span class="math display">\[x= (1, 2, 3)^T\]</span></p>
+<h2 id="普通算数操作符">普通算数操作符</h2>
 <ul>
-<li>那么它的二范数表示如下</li>
+<li><code>__add__(self, other)</code>: 实现加法</li>
+<li><code>__sub__(self, other)</code>: 实现减法</li>
+<li><code>__mul__(self, other)</code>: 实现乘法</li>
+<li><code>__floordiv__(self, other)</code>: 实现整除法</li>
+<li><code>__div__(self, other)</code>: 实现除法</li>
+<li><code>__mod__(self, other)</code>: 实现取模</li>
+<li><code>__pow__(self, other)</code>: 实现指数</li>
+<li><code>__lshift__(self, other)</code>: 实现&lt;&lt;</li>
+<li><code>__rshift__(self, other)</code>: 实现&gt;&gt;</li>
+<li><code>__and__(self, other)</code>: 实现&amp;</li>
+<li><code>__or__(self, other)</code>: 实现|</li>
+<li><code>__xor__(self, other)</code>: 实现^</li>
 </ul>
-<p><span class="math display">\[||x||_2 =\sqrt{1^2 + 2^2 +
-3^2}=\sqrt{14}\]</span></p></li>
-<li><p>共轭转置</p>
-<p><span class="math display">\[A^*=(\bar{A})^T=\bar{A^T}\]</span></p>
+<h2 id="其他">其他</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.name = value</span><br><span class="line">    <span class="comment">#每当属性被赋值的时候， ``__setattr__()`` 会被调用，这样就造成了递归调用。</span></span><br><span class="line">    <span class="comment">#这意味这会调用 ``self.__setattr__(&#x27;name&#x27;, value)`` ，每次方法会调用自己。这样会造成程序崩溃。</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.__dict__[name] = value  <span class="comment">#给类中的属性名分配值</span></span><br><span class="line">    <span class="comment">#定制特有属性</span></span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">AccessCounter</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;一个包含计数器的控制权限的类每当值被改变时计数器会加一&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, val</span>):</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="number">0</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;value&#x27;</span>, val)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">    <span class="comment">#如果你不想让其他属性被访问的话，那么可以抛出 AttributeError(name) 异常</span></span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(name, value)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delattr__</span>(<span class="params">self, name</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__delattr__(name)]</span><br></pre></td></tr></table></figure>
 <ul>
-<li>现有矩阵A</li>
+<li><p><code>__str__(self)</code>: 返回值，人类可读</p></li>
+<li><p><code>__repr__(self)</code>: 返回值，机器可读</p></li>
+<li><p><code>__unicode__(self)</code>: 返回unicode()</p></li>
+<li><p><code>__hash__(self)</code>: hash()返回一个整形</p></li>
+<li><p><code>__nonzero__(self)</code>: bool()返回的值</p></li>
+<li><p><code>__getattr__(self, name)</code>:
+当用户试图获取一个不存在（或不建议的属性）的属性时，给出一些警告</p></li>
+<li><p><code>__setattr__(self, name, value)</code>:
+无论属性是否存在，都允许你定义对属性的赋值行为</p></li>
+<li><p><code>__delattr__(self, name)</code>： 删除一个属性</p></li>
 </ul>
-<p><span class="math display">\[A=\left[ \begin{array}{ccc} 3+i &amp; 5
-\\ 2-2i &amp; i\end{array} \right]\]</span></p>
+<h2 id="封装附加魔术方法">封装附加魔术方法</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">FunctionalList</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;一个封装了一些附加魔术方法比如 head, tail, init, last, drop, 和take的列表类。</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, values=<span class="literal">None</span></span>):</span><br><span class="line">        <span class="keyword">if</span> values <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = []</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = values</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__len__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__getitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="comment">#如果键的类型或者值无效，列表值将会抛出错误</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setitem__</span>(<span class="params">self, key, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values[key] = value</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__iter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">iter</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__reversed__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">reversed</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">append</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values.append(value)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">head</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tail</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">1</span>:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">init</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回一直到末尾的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">last</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回末尾元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">drop</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回除前n个外的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[n:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">take</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回前n个元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:n]</span><br></pre></td></tr></table></figure>
+<h2 id="反射">反射</h2>
 <ul>
-<li>则它的共轭矩阵如下</li>
+<li><code>__instancecheck__(self, instance)</code>:
+检查一个实例是否为定义类的实例</li>
+<li><code>__subclasscheck__(self, subclass)</code>:
+检查一个类是否为定义类的子类</li>
 </ul>
-<p><span class="math display">\[A^*=\left[ \begin{array}{ccc} 3-i &amp;
-2+2i \\ 5 &amp; -i\end{array} \right]\]</span></p></li>
-<li><p>特征值与奇异值</p>
-<p>我的理解是：特征值是奇异值的一种特殊情况，特征值是在矩阵是方阵时候的奇异值。</p>
-<p>关于什么是特征值。</p>
-<p>一个矩阵可以理解为由n个向量组成的，而特征值就是反应向量长度（重要性）的标准，特征值越大，则对应向量就越重要。</p>
-<p>假设判断电影为动作片还是搞笑片，可以由以下几个特征来衡量，打斗场景、观众笑声次数、导演、演员……。将数据离散化成矩阵，然后提取特征值，特征值越大的特征则对于衡量电影是动作片还是搞笑片的比重越大。</p></li>
-<li><p>Householder矩阵</p>
+<h2 id="更改状态">更改状态</h2>
 <ul>
-<li><p><a
-href="https://zh.wikipedia.org/wiki/豪斯霍尔德变换">wiki链接</a></p></li>
-<li><p>公式如下，H表示Householder矩阵，I表示单位矩阵，v表示单位向量，v*表示v的共轭转置。</p>
-<p><span class="math display">\[H=I-2vv^*\]</span></p></li>
-<li><p>性质：</p>
-<ol type="1">
-<li>H矩阵的转置等于它本身的逆</li>
-<li>H矩阵的平方等于单位矩阵</li>
-</ol>
-<p><span class="math display">\[H^{-1}=H^*\]</span></p>
-<p><span class="math display">\[H^2=I\]</span></p></li>
-</ul></li>
+<li><code>__call__(self, [...])</code>:
+经常改变状态的时候，调用这个方法是一种直接和优雅的方法</li>
 </ul>
-<h3 id="svd分解">1. SVD分解</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Entity</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;调用实体来改变实体的位置。&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, size, x, y</span>):</span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br><span class="line">        <span class="variable language_">self</span>.size = size</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__call__</span>(<span class="params">self, x, y</span>):</span><br><span class="line">        <span class="string">&#x27;&#x27;&#x27;改变实体的位置&#x27;&#x27;&#x27;</span></span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br></pre></td></tr></table></figure>
+<h2 id="会话管理">会话管理</h2>
+<p>使用with语句来调用</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Closer</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;通过with语句和一个close方法来关闭一个对象的会话管理器&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, obj</span>):</span><br><span class="line">        <span class="variable language_">self</span>.obj = obj</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__enter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.obj <span class="comment"># bound to target</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__exit__</span>(<span class="params">self, exception_type, exception_val, trace</span>):</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="variable language_">self</span>.obj.close()</span><br><span class="line">        <span class="keyword">except</span> AttributeError: <span class="comment"># obj isn&#x27;t closable</span></span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&#x27;Not closable.&#x27;</span>)</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">True</span> <span class="comment"># exception handled successfully</span></span><br></pre></td></tr></table></figure>
+<h2 id="slots__"><code>__slots__</code></h2>
+<p>限制class的属性，但不限制class的子类的属性，除非子类也使用<code>__slots__</code></p>
+<p>例子：</p>
+<p>使用<code>__slots__</code>会比不使用，减少40%~50%的内存</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 不使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  __slots__ = [<span class="string">&#x27;name&#x27;</span>, <span class="string">&#x27;identifier&#x27;</span>]</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Magic Methods</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>多进程嵌套内存泄漏问题记录</title>
+    <url>/hexo_blog/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>记一次多进程嵌套发现的内存泄漏问题，以及解决方案。</p>
+<span id="more"></span>
+<h3 id="问题代码">问题代码</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> multiprocessing</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ProcessPoolExecutor</span><br><span class="line"><span class="keyword">import</span> concurrent.futures</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Any</span></span><br><span class="line"><span class="keyword">import</span> gc</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Foo</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.executor = ProcessPoolExecutor(max_workers=<span class="number">10</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>))</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">fun</span>(<span class="params">arg</span>):</span><br><span class="line">        <span class="keyword">raise</span> concurrent.futures.process._RemoteTraceback(<span class="string">&quot;error&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">        x = []</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">            res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="keyword">with</span> ProcessPoolExecutor(max_workers=<span class="number">2</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>)) <span class="keyword">as</span> e:</span><br><span class="line">        futures = e.submit(predict)</span><br><span class="line">        futures.result()</span><br></pre></td></tr></table></figure>
+<p>肉眼观察 乍一看这段代码是没有任何问题的，在 submit 之后，获取 result
+，并且用 try ... except ... 来捕获可能出现的任何情况。尤其是，当不使用
+main 函数中的进程池时，直接普通的使用 predict
+函数，这段代码不会引发任何问题，可以正常的退出。</p>
+<p>但为什么改成多进程时，这段代码就无法正常退出了呢？程序无法正常退出往往是因为程序某些对象没有完全被释放干净，但这里并没有使用特殊的函数却依旧引发了该问题，所以需要借助工具来进行简单的分析。</p>
+<h3 id="分析">分析</h3>
+<p>garbage collector python 的垃圾回收机制主要使用引用计数（reference
+counting）来跟踪和回收垃圾。大部分情况下，Python
+能够在对象结束执行后，引用计数会变为
+0，使得最后能够被回收。这里程序退出不了的原因是某些对象没发自动退出，所以在
+predict 函数中，观察变量 f 和 f.executor 的引用计数</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
+<p>程序的输出，对象 f 的计数为 5，而 f.executor 的计数为 2。为什么是
+2，因此在最外层还有一个进程数量为 2 的进程池没有执行完。而 f 等于 4
+而不等于 f.executor 这就不符合预期，比正常数值多 3。这个 3 是因为循环了
+3 次，每次都引发了异常。</p>
+<p>进一步地，观察 f 中什么对象没有被清理。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(gc.get_referrers(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
+<p>此时输出会多一行，形如</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">[&lt;frame at 0x10c266240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0440, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;]</span><br></pre></td></tr></table></figure>
+<p>这表明程序在何处的对象没有被处理，可以定位到函数 call 的 return
+位置。</p>
+<p>为什么？ 在获取 result 时，如果遇到异常，则中间变量 future
+无法被回收。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">result</span>(<span class="params">self, timeout=<span class="literal">None</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;Return the result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Args:</span></span><br><span class="line"><span class="string">        timeout: The number of seconds to wait for the result if the future</span></span><br><span class="line"><span class="string">            isn&#x27;t done. If None, then there is no limit on the wait time.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Returns:</span></span><br><span class="line"><span class="string">        The result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Raises:</span></span><br><span class="line"><span class="string">        CancelledError: If the future was cancelled.</span></span><br><span class="line"><span class="string">        TimeoutError: If the future didn&#x27;t finish executing before the given</span></span><br><span class="line"><span class="string">            timeout.</span></span><br><span class="line"><span class="string">        Exception: If the call raised then that exception will be raised.</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">with</span> <span class="variable language_">self</span>._condition:</span><br><span class="line">            <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line"></span><br><span class="line">    <span class="variable language_">self</span>._condition.wait(timeout)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">raise</span> TimeoutError()</span><br><span class="line">    <span class="keyword">finally</span>:</span><br><span class="line">        <span class="comment"># Break a reference cycle with the exception in self._exception</span></span><br><span class="line">        <span class="variable language_">self</span> = <span class="literal">None</span></span><br></pre></td></tr></table></figure>
+<h3 id="解决方案">解决方案</h3>
+<h4 id="消除中间变量-future">消除中间变量 future</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 改为</span></span><br><span class="line">res = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i).result(timeout=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<h4 id="手动-gc">手动 GC</h4>
+<p>在调用完 Foo 之后，使用 gc.collect()</p>
+<h4 id="主动关闭进程池">主动关闭进程池</h4>
+<p>在调用完 Foo 之后，使用 f.executor.shutdown()</p>
+<h4 id="在-return-前-主动删除未释放的对象">在 return 前
+主动删除未释放的对象</h4>
+<p>如下所示，使用 future = None 也可以达到相似效果（让 Python
+自动调用垃圾回收机制）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">   x = []</span><br><span class="line">   <span class="keyword">try</span>:</span><br><span class="line">       future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">       res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">   <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">       <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">       <span class="keyword">del</span> future</span><br><span class="line">   <span class="keyword">return</span> x</span><br></pre></td></tr></table></figure>
+<h3 id="q-a">Q &amp; A</h3>
+<p>Q: 能不能在 Foo 中添加 <strong>del</strong> 函数解决这个问题 A: 在
+Python 3.8.16 可以，Python 3.9.6 不行。可以手动增加 <strong>del</strong>
+函数添加 print 函数，观察是否有对应的输出。</p>
+<p>Q: gc 显示为什么是在 return 这，而不是其他地方？ A: frame
+对象有一个属性叫作 f_lineno，它表示当前执行的代码行。当函数执行到 return
+语句时，f_lineno 会更新为 return 语句所在的行号。</p>
+<p>这是因为在函数返回值之后，控制权会返回到调用该函数的代码。此时函数的上下文（包括它的局部变量、执行状态等）会被丢弃，栈帧也会被弹出调用栈。这个过程称为函数的退出或返回。所以，return
+语句所在的行是函数执行的最后一行。</p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>多进程</tag>
+        <tag>内存泄漏</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python装饰器</title>
+    <url>/hexo_blog/2018/01/30/Python/Python%E8%A3%85%E9%A5%B0%E5%99%A8/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Python的装饰器是什么和一些Python内置装饰器的使用</p>
+<span id="more"></span>
+<p>参考文章:</p>
 <p><a
-href="https://blog.csdn.net/u010099080/article/details/68060274">数学解释参考链接</a></p>
-<p>奇异值分解。公式如下：</p>
-<p><span class="math display">\[M=U\Sigma V^*\]</span></p>
-<p>举例：</p>
-<p><span class="math display">\[\begin{array}{clll} M &amp;= U\Sigma V^*
-\\ &amp;= \begin{bmatrix} 1 &amp; 0 &amp; 0 &amp; 0 &amp; 2\\ 0 &amp; 0
-&amp; 3 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 4
-&amp; 0 &amp; 0 &amp; 0\end{bmatrix} \\  &amp;= \begin{bmatrix} 0 &amp;
-0 &amp; 1 &amp; 0\\ 0 &amp; 1 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp;
-1\\ 1 &amp; 0 &amp; 0 &amp; 0\end{bmatrix} \cdot \begin{bmatrix} 4 &amp;
-0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 3 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp;
-0 &amp; \sqrt{5} &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp;
-0\end{bmatrix} \cdot \begin{bmatrix} 0 &amp; 1 &amp; 0 &amp; 0 &amp; 0\\
-0 &amp; 0 &amp; 1 &amp; 0 &amp; 0\\ \sqrt{0.2} &amp; 0 &amp; 0 &amp; 0
-&amp; \sqrt{0.8}\\ 0 &amp; 0 &amp; 0 &amp; 1 &amp; 0\\ \sqrt{0.8} &amp;
-0 &amp; 0 &amp; 0 &amp; -\sqrt{0.2}\end{bmatrix}
-\end{array}\]</span></p>
-<p>性质:</p>
-<ol type="1">
-<li>矩阵<span
-class="math inline">\(\Sigma\)</span>的所有非对角元为0。矩阵<strong>U</strong>和<span
-class="math inline">\(V^*\)</span>都是酉矩阵，它们乘上各自的共轭转置都得到单位矩阵</li>
-<li>奇异值的分解并不是唯一的，这是因为<span
-class="math inline">\(\Sigma\)</span>矩阵中的对角元有元素为0</li>
-</ol>
-<p>步骤：</p>
-<ol type="1">
-<li>计算<span class="math inline">\(M.T\cdot
-M\)</span>得到特征向量（左奇异向量），是U的列向量</li>
-<li>计算<span class="math inline">\(M\cdot
-M.T\)</span>得到特征向量（右奇异向量），是V的列向量</li>
-<li>计算<span class="math inline">\(M\cdot M.T\)</span>或者<span
-class="math inline">\(M.T\cdot
-M\)</span>得到特征值，所有非零特征值的平方根（二范数）</li>
-</ol>
-<p>得到特征向量之后，需要依照特征值从大到小进行排序。</p>
-<p>关于求特征值的方法，线代书上已做介绍，就不赘述。如果运用编程方法求矩阵的特征值，最常见的方法有<strong>雅可比算法、幂法。</strong>当然也可以用先进行QR分解求得特征值之后，再继续进行SVD。</p>
-<h3 id="lu分解">2. LU分解</h3>
+href="https://www.zhihu.com/question/26930016">如何理解Python装饰器？</a></p>
 <p><a
-href="https://jiayi797.github.io/2017/03/29/%E6%95%B0%E5%AD%A6-%E7%9F%A9%E9%98%B5LU%E5%88%86%E8%A7%A3/">参考链接</a></p>
-<p>将一个矩阵分解为一个下三角矩阵和一个上三角矩阵的乘积。形式如：</p>
-<p><span class="math display">\[A=LU\]</span></p>
-<p>格式如下</p>
-<p><span class="math display">\[\left[ \begin{array} { l l l } { a _ {
-11 } } &amp; { a _ { 12 } } &amp; { a _ { 13 } } \\ { a _ { 21 } } &amp;
-{ a _ { 22 } } &amp; { a _ { 23 } } \\ { a _ { 31 } } &amp; { a _ { 32 }
-} &amp; { a _ { 33 } } \end{array} \right] = \left[ \begin{array} { c c
-c } { l _ { 11 } } &amp; { 0 } &amp; { 0 } \\ { l _ { 21 } } &amp; { l _
-{ 22 } } &amp; { 0 } \\ { l _ { 31 } } &amp; { l _ { 32 } } &amp; { l _
-{ 33 } } \end{array} \right] \left[ \begin{array} { c c c } { u _ { 11 }
-} &amp; { u _ { 12 } } &amp; { u _ { 13 } } \\ { 0 } &amp; { u _ { 22 }
-} &amp; { u _ { 23 } } \\ { 0 } &amp; { 0 } &amp; { u _ { 33 } }
-\end{array} \right]\]</span></p>
-<p>分解过程：</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/matrix/2017-03-29-20-13-15.png" /></p>
-<p>声明：l11表示L矩阵的第一行第一列的元素，U1i表示L矩阵第一行每一列（需要遍历的列）的元素</p>
+href="https://www.cnblogs.com/cicaday/p/python-decorator.html">详解Python的装饰器</a></p>
+<h2 id="装饰器是什么">装饰器是什么</h2>
+<p>python的装饰器本身就是一个函数，使用它可以在某些场景下简化python代码，增加一些额外功能。我的理解是，python装饰器可以帮助同时多个函数做一样的事情（比如打印日志），而避免了每个函数都需要额外加上一样的代码。</p>
+<p>举个例子，假设我们有两个函数</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>以上一段代码片段很容易理解，运行结果就是两行输出。那么问题来了，如果这个时候我需要在每个函数里面加上一个输出，比如<code>print("this is func")</code>。很简单，我们想到了第一种解决方案就是同时在两个函数里面加上这么一行。</p>
+<p>但是如果我们需要在很多个函数在上很多个这种重复代码，我们这么做就很浪费时间了，而且容易出错。这个时候我们想到了第二种解决方案。定义一个新的函数，在每次使用函数之前，先调用新函数。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">    func()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_print(func_one)</span><br><span class="line">    func_print(func_two)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>感觉不错，但是如果这样无疑改变了原来的逻辑结构，且每次需要传递函数给<code>func_print</code>。下面就轮到我们的主角登场了——装饰器。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one = func_print(func_one)</span><br><span class="line">    func_two = func_print(func_two)</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>这样也能达到我们的想要的效果，但是比之前的实现方式更难看懂。而且调用起来似乎更复杂了。</p>
+<p>第一个问题：更难看懂</p>
+<p>可能是对<code>*args</code>、<code>**kwargs</code>的参数理解不是很好。简单解释一下就是说,<code>*args</code>可以理解为一个<code>list</code>，接收多个参数。<code>**kwargs</code>可以理解为一个<code>dict</code>，接收多个<code>key</code>和<code>value</code>。接收的参数数量不定，可以没有，也可以很多个。再具体一点可以百度看看。</p>
+<p>第二个问题：调用更复杂</p>
+<p>其实上面的例子其实调用装饰器的一种方式，<code>python</code>有<code>@</code>的这个语法糖，所以一般情况下，我们是像下面一样调用的。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>好的，简单的装饰器调用基本就是这样，难度进阶，如何使用带参数的装饰器。换句话就说，如果我们需要根据每个函数自定义一些东西，如何传递参数给装饰器。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func_name</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">decorator</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">        <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;this is %s&quot;</span> % func_name)</span><br><span class="line">            <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">        <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> decorator</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params"><span class="string">&quot;one&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params">func_name=<span class="string">&quot;two&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>观察<code>func_print</code>部分代码，与之前的不同在于多加了一个<code>decorator</code>函数封装了<code>wrapper</code>函数。我的理解是这样的：<code>func_print</code>作用是装饰器的名字和接收外部的参数，<code>decorator</code>接收外部的函数，<code>wrapper</code>返回结果给函数。<code>decorator</code>翻译为中文是<strong>装饰</strong>。<code>wrapper</code>翻译过来是<strong>包装</strong>。<code>decorator</code>装饰函数，<code>wrapper</code>包装函数。</p>
+<p>当然，<code>decorator</code>和<code>wrapper</code>这两个函数名字可以自定义，在这里为了好理解所以用了这两个名字</p>
+<h2 id="内置装饰器">内置装饰器</h2>
+<h3 id="property"><code>@property</code></h3>
+<p>如果要了解使用这个装饰器，就需要知道在不用装饰器的情况下，写一个属性。这里使用python自带的一个例子。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">getx</span>(<span class="params">self</span>): <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">setx</span>(<span class="params">self, value</span>): <span class="variable language_">self</span>._x = value</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">delx</span>(<span class="params">self</span>): <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br><span class="line">    x = <span class="built_in">property</span>(getx, setx, delx, <span class="string">&quot;I&#x27;m the &#x27;x&#x27; property.&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>上面就是定义一个python属性的标准写法，下面使用<code>@property</code>来定义属性</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="string">&quot;I am the &#x27;x&#x27; property.&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line"><span class="meta">    @x.setter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>._x = value</span><br><span class="line"><span class="meta">    @x.deleter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br></pre></td></tr></table></figure>
+<p>这样有什么好处呢，原来的定义，属性访问并没有做限制。使用<code>@property</code>可以使<code>x</code>这个属性更加安全，从获取、设置必须与属性名一致。</p>
+<p>Emmmm.....我的理解就是加上<code>@property</code>的话，定义属性会更加安全更加优雅。毕竟是面向对象了，天知道你的用户会对你的属性做一些什么奇奇怪怪的事情。</p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Decorators</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>海量数据处理问题（HashMap+分治法、位图法）</title>
+    <url>/hexo_blog/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</url>
+    <content><![CDATA[<p>从海量数据中找出不重复的数和出现次数最多的数的解决方案，分别采用了分治法+HashMap和位图法Bitmap两种方法。其中，位图法在大数据量下处理速度更快，内存占用更小。而对于其他类似问题，可以尝试使用分治法加最小堆/最大堆的方式处理。</p>
+<span id="more"></span>
+<p>原文链接：<a
+href="https://juejin.im/post/5aa0ee9f518825557c010bc0">经典面试问题: Top
+K 之 ---- 海量数据找出现次数最多或，不重复的。</a></p>
+<p>在数据处理中，较为棘手的有这么一种问题，海量数据如何进行处理。一旦数据的数量足够大，我们就不能使用常规的思路来解决。典型例子就是排序算法，在数据不大（不超过内存）的情况下，可以直接使用排序算法，但是当数据量很大时，我们就需要使用如多路归并此类的算法。</p>
+<p>总结问题：</p>
 <ol type="1">
-<li>初始化L、U矩阵。L矩阵下三角矩阵，对角元素全为1；U矩阵为上三角矩阵。</li>
-<li>计算U矩阵的第一行，对应A矩阵的第一行元素，不需要做任何操作</li>
-<li>计算L矩阵的第一列（除去第一行元素），对应A矩阵的第一列元素，每个元素除以u11</li>
-<li>计算U矩阵的第二行（除去第一列元素），对应A矩阵的第二行元素，每个元素减去l21*u1i</li>
-<li>计算L矩阵的第二列（除去前两行元素），对应A矩阵的第二列元素，每个元素先减去l31*u12，再除以u22</li>
+<li>内存一次性不能放下所有的数</li>
+<li>机器CPU核数不够</li>
 <li>……</li>
 </ol>
-<p>总结公式如下：</p>
-<p>L矩阵对角元素为1。</p>
-<p>第一步的公式：</p>
-<p><span class="math display">\[\begin{array}{lll}
-a_{1j}=(1,0,0,\cdots,0)\cdot(u_{1j},u_{2j},\cdots,u_{ij}).T=u_{1j} \\
-a_{i1}=(l_{i1},
-l_{i1},\cdots,l_{i1})\cdot(u_{11},0,\cdots,0).T=l_{i1}\cdot u_{11}
-\\u_{1j}=a_{1j}  \\ l_{i1}=\frac{a_{i1}}{u_{11}} \\j=1,2,\cdots,n
-\\i=2,3,\cdots,n\end{array}\]</span></p>
-<p>第r步的公式：</p>
-<p><span
-class="math display">\[\begin{array}{rl}  a_{rj}&amp;=(l_{r1},l_{r2},...,l_{rr-1})\cdot(u_{1j},u_{2j},...,u_{jj}).T
-\\&amp;=\sum\limits_{k=1}^{r-1}l_{rk}u_{kj}+u_{rj}  \end{array}\]</span></p>
-<p><span class="math display">\[\begin{array}{lll}
-u_{rj}=a_{rj}-\sum\limits_{k=1}^{r-1}l_{rk}u_{kj} \\
-l_{ir}=\frac{a_{ir}-\sum\limits_{k=1}^{r-1}l_{ik}u_{kr}}{u_{rr}}
-\end{array}\]</span></p>
-<p><span class="math display">\[\begin{array}{lll} j=r,r+1,\cdots,n \\
-r=2,3,\cdots,n-1 \\ i=r+1,\cdots,n\end{array}\]</span></p>
-<h3 id="qr分解">3. QR分解</h3>
-<p><a
-href="https://wenku.baidu.com/view/c2e34678168884868762d6f9.html">参考链接</a></p>
-<p>把矩阵分解成一个半正交矩阵与一个上三角矩阵的积。</p>
-<p>形式如:</p>
-<p><span class="math display">\[A=QR\]</span></p>
-<p>分解定理：任意一个满秩实（复）矩阵A，都可唯一地分解A=QR,其中Q为正交（酉）矩阵，R是具有对角元的上三角矩阵。</p>
-<h4 id="三种实现方式">三种实现方式</h4>
-<ol type="1">
-<li><p>Gram-Schmidt正交化</p>
-<p>过程:</p>
-<ol type="1">
-<li><p>取出矩阵A中的每一列命名为x_col，它们之间线性无关</p></li>
-<li><p>将每一个x_col正交化得到y_col</p></li>
-<li><p>再将y_col进行单位化，即转换为e_col。这里每个e_col为矩阵Q中的每一列。</p></li>
-<li><p>将第二步中，先转换为y_col的正交化得到x_col，再转换为e_col正交化得到x_col。这里每个e_col的系数，对应的就是矩阵R中的每一列</p></li>
-</ol></li>
-<li><p>Householder变换</p>
-<p><a href="https://www.bbsmax.com/A/gVdnKqK1zW/">参考链接</a></p>
-<p>outer函数</p>
-<ul>
-<li><p>通常有两个输入向量，假设为向量a、b</p>
-<p><span class="math display">\[a = (1, 2, 3) , b = (-1, -2,
--3)\]</span></p></li>
-<li><p>那么，outer函数运算过程如</p></li>
-</ul>
-<p><span class="math display">\[outer(a, b) = \left[ \begin{array}{ccc}
-1*-1 &amp; 1*-2 &amp; 1*-3 \\ 2*1 &amp; 2*-2 &amp; 2*-3 \\ 3*-1 &amp;
-3*-2 &amp; 3*-3 \end{array} \right]\]</span></p>
-<p>过程:</p>
+<h2 id="不重复数">不重复数</h2>
+<p>在2.5亿个正整数中找出不重复的数。</p>
+<h3 id="分治法hashmap">分治法+HashMap</h3>
+<h4 id="说明">说明</h4>
+<p>比如将这些数据平均分为100个批次，每个批次就有250万数。每个批次用循环进行遍历，存入
+<code>HashMap&lt;a, b&gt;</code>，<code>a</code>表示每个数，<code>b</code>表示对应数出现次数，默认为1。每遍历完一批，就对当前
+<code>HashMap</code>进行去重，如果
+<code>b &gt; 1</code>，就排除此数。全部批次进行完筛选之后，剩下的数最后可以直接进行统计，得出不重复的数。</p>
+<h4 id="双间复杂度">双间复杂度</h4>
+<p>时间复杂度：250W * 100轮 + 其他批次。
+这里可以使用多核机器加速运算。</p>
+<p>空间复杂度：int保存数，<code>Key + Value（250W * 4Byte）/ (1024 * 1024)</code>约为
+<code>(Key + 9.5M)</code>内存</p>
+<h3 id="位图法-bitmap">位图法 Bitmap</h3>
+<h4 id="说明-1">说明</h4>
+<p>设计每两个
+<code>bit</code>位，标识一个数的出现情况。如：00表示没有出现，01表示出现一次，10表述出现多次。此题给出的是正整数，所以就是无符号的整型数，占四个字节。</p>
+<p>内存的计算。1B=8b, 4B=32b, 它可以表示的最大整数是<span
+class="math inline">\(2^{32}-1\)</span>, 即需要$2^{32}-1 $约位<span
+class="math inline">\(2^{32}\)</span>个<strong>位</strong>来表示这2.5亿个数，每个状态是两个数，总共就是<span
+class="math inline">\(2^{32}\times 2\)</span>个位。</p>
+<p>一次性申请的位图内存是: $2^{32} $ bit, 即<span
+class="math inline">\(\frac{2^{32}\times2}{(1024\times1024 \times
+8)}=1\)</span> GB。如果加上分治法，甚至都不需要1G内存。</p>
+<p>假设一个数是64, 则对应的bit位是 <span class="math inline">\(64\times
+2=128\)</span>，即127和128共同标识这个数的出现状态（没有出现、出现一次、出现多次）。所以，每当读入了一个数，先找对应的bit位，如果是00就改为01，如果是01就改为10，如果是10就不需要修改。最后扫描整个位图，如果是10，就将下标/2得出出现次数为1的这个数。</p>
+<h2 id="出现次数最多的">出现次数最多的</h2>
 <ol type="1">
-<li>初始化矩阵Q为单位矩阵(rxr)，r为矩阵A的行数。初始化矩阵R为矩阵A</li>
-<li>取R中的下三角矩阵，每次取一列命名为x（不包括最后一列）。e初始化为形如x的零向量</li>
-<li>e中的第一个元素为向量x的二范数</li>
-<li>这里得到v，范数取2</li>
-</ol>
-<p><span class="math display">\[v = \frac{x - e}{||x-e||}\]</span></p>
-<ol start="5" type="1">
-<li>初始化一个单位矩阵命名为Q_cnt，取Q_cnt的下三角矩阵，每次取一列在原有的基础上减去2*outer(v,
-v)。（不包括最后一列）</li>
-<li>更新R,Q</li>
-</ol>
-<p><span class="math display">\[R = Q\_cnt * R \]</span></p>
-<p><span class="math display">\[Q = Q * Q\_cnt\]</span></p></li>
-<li><p>Givens变换</p>
-<p><a
-href="http://www.voidcn.com/article/p-vmnqitql-xa.html">参考链接</a></p>
-<p>有待补充</p></li>
+<li>找出一篇文章中，出现次数最多的单词</li>
+<li>10亿个正整数找出重复次数最多的100个整数</li>
 </ol>
+<p>这类问题被叫做Top K问题</p>
+<h3 id="分治法hashmap-1">分治法+HashMap</h3>
+<p>对于文章来说，我们需要规定每批次的字数限制。这里假设100为一批的处理极限，每次处理100个，如果最后一批少于100个就直接当作100来处理。之后每批文字进行切分(<code>split</code>)，比如英文的切割方法可以是空格、标点符号什么的，中文的切割方式可以是空格或者有预先准备好的词库。</p>
+<p>之后就跟上面不重复数字的思路类似了，遍历每批数据，存入
+<code>HashMap&lt;String, Integer&gt;</code>，String表示字符串，Integer对应出现的次数，最大的Integer就是出现次数最多的。</p>
+<p>对数字来说，就是跟不重复数字思路类似，将十亿个数分乘很多份，利用
+<code>HashMap&lt;int, int&gt;</code>来统计，最后找到第二个int最大的那个。每批筛选完之后，就进行排序，选出十亿个数中最大的。</p>
+<h3 id="位图法-bitmap-1">位图法 Bitmap</h3>
+<p>对于文章（字符串）来说，不建议采用</p>
+<p>对于数字来说，此题的背景不需要乘2，将每个数存进位图。出现了一次，bit位=1，没有出现就是0。但是没有办法统计出现次数，所以此题也不适合采用位图法。</p>
+<h2 id="其他">其他</h2>
+<p>在XXX中找出最大、最小、最大的几个、最小的几个。都可以使用分治法+最小堆/最大堆。</p>
 ]]></content>
       <categories>
         <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>数学</tag>
-        <tag>矩阵分解</tag>
+        <tag>HashMap</tag>
+        <tag>位图法</tag>
       </tags>
   </entry>
   <entry>
-    <title>线性代数中各种矩阵的简介</title>
-    <url>/hexo_blog/2019/05/13/%E6%95%B0%E5%AD%A6/%E7%BA%BF%E4%BB%A3%E7%9F%A9%E9%98%B5/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了线性代数的矩阵，仅作笔记。</p>
+    <title>Leetcode 字符串刷题记录</title>
+    <url>/hexo_blog/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</url>
+    <content><![CDATA[<p>字符串刷题分类</p>
 <span id="more"></span>
-<p>方阵：行数列数相同的矩阵</p>
-<p>对称矩阵：<span
-class="math inline">\(A=A^T\)</span>的矩阵，矩阵等于它自己的转置矩阵</p>
-<p>反对称矩阵：<span class="math inline">\(A=-A^T\)</span></p>
-<p>埃尔米特矩阵（Hermitian matrix）：<span
-class="math inline">\(A=A^*\)</span>，矩阵等于它自己的共轭转置矩阵</p>
-<p>斜埃尔米特矩阵：反埃尔米特矩阵，<span class="math inline">\(A =
-−A^*\)</span></p>
-<p>正定矩阵：<span class="math inline">\(Q(x)=x^T A
-x\)</span>，任意非零向量<span
-class="math inline">\(x\)</span>，若满足<span
-class="math inline">\(Q(x)&gt;0\)</span>，则<span
-class="math inline">\(A\)</span>为正定矩阵</p>
-<p>半正定矩阵：同上，<span class="math inline">\(Q(x) \geq
-0\)</span>，则A为半正定矩阵</p>
-<p>半负定矩阵：同上，<span class="math inline">\(Q(x) \leq
-0\)</span>，则A为半负定矩阵</p>
-<p>不定矩阵：若A既不是半正定矩阵，也不是半负定矩阵，则A为不定矩阵</p>
-<p>上三角矩阵：矩阵对角线左下方系数全为0</p>
-<p>下三角矩阵：矩阵对角线右上方系数全为0</p>
-<p>三对角矩阵：非零系数在主对角线上，也在比主对角线低和高一行的对角线上。</p>
-<p>单位矩阵：<span class="math inline">\(I\)</span>,
-矩阵对角线元素为1，其余为0</p>
-<p>高斯矩阵：首先满足单位矩阵，其次某列系数非0</p>
-<p>逆矩阵：反矩阵，<span
-class="math inline">\(AB=I\)</span>，A是可逆的，B是A的逆矩阵</p>
-<p>伴随矩阵：<strong>A</strong>关于第<em>i</em>行第<em>j</em>列的<strong>余子式</strong>，是去掉<strong>A</strong>的第<em>i</em>行第<em>j</em>列之后得到的(<em>n</em>
-− 1)×(<em>n</em> − 1)矩阵的行列式。</p>
-<p>阶梯型矩阵：从上至下，<span
-class="math inline">\(A_{ii}\)</span>之前的元素为0</p>
-<p>酉矩阵：<span class="math inline">\(UU^*=U^*U=I\)</span>，<span
-class="math inline">\(U^*\)</span>为U矩阵的共轭转置，则U为酉矩阵</p>
-<p>对角矩阵：只有矩阵中的对角线存在值，其他地方为0</p>
-<p>空矩阵：行数或列数为零的矩阵</p>
-<p>分块矩阵：是指一个大矩阵分割成“矩阵的矩阵“</p>
-<p>正交矩阵：满足方阵，<span
-class="math inline">\(Q^T=Q^{-1}\)</span>，<span
-class="math inline">\(Q^TQ=QQ^T=I\)</span>，正交矩阵的行列式为1</p>
-<p>旋转矩阵：在乘以一个向量的时候改变向量的方向但不改变大小的效果并保持了手性的矩阵</p>
-<p>正规矩阵：<span class="math inline">\(A^{*}A=AA^*\)</span>，<span
-class="math inline">\(A^*\)</span>为<span
-class="math inline">\(A\)</span>的共轭矩阵</p>
-<p>共轭矩阵：<span class="math inline">\(A^*=(\bar{A})^T\)</span>，<span
-class="math inline">\(\bar{A}\)</span>表示矩阵的复共轭</p>
-<p>奇异矩阵：满足方阵，矩阵的行列式是否等于0，若等于0，称矩阵A为奇异矩阵</p>
-<p>非奇异矩阵：同上，若不等于0，称矩阵A为非奇异矩阵。</p>
-<p>稀疏矩阵：若数值为0的元素数目远远多于非0元素的数目，并且非0元素分布没有规律时</p>
-<p>稠密矩阵：与上相反，若非0元素数目占大多数时</p>
-<p>增广矩阵：扩增矩阵，在系数矩阵的右边添上一列，这一列是线性方程组的等号右边的值</p>
-<p>系数矩阵：将方程组的系数组成矩阵来计算方程的解</p>
-<p>相似矩阵：判断特征值、行列式、迹、秩是否相等，只作为判断矩阵是否相似的必要条件，而非充分条件。</p>
+<h2 id="滑动窗口">滑动窗口</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-substring-without-repeating-characters/">3.
+无重复字符的最长子串</a></p>
+<p>从左向右滑，需要一个数据结构（A）记录已经遍历的字符，还有一个变量（B）记录最大长度</p>
+<p>注：每次不需要从头开始，而是可以从重复那个字符的后一个开始，所以A需要记录一下索引</p>
+<h2 id="kmp">KMP</h2>
+<h2 id="动态规划">动态规划</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-palindromic-substring/">5.
+最长回文子串</a></p>
+<p>两个指针从两头开始，所以需要二维DP，元素是bool判断</p>
+<p><a href="https://leetcode.cn/problems/interleaving-string/">97.
+交错字符串</a></p>
+<p>两个字符串组成新的字符串，二维DP，元素是bool判断</p>
+<p><a href="https://leetcode.cn/problems/word-break/">139.
+单词拆分</a></p>
+<p>单词数组，一个一个单词拼接，组成最后单词，一维DP，元素是bool判断</p>
+<p><a
+href="https://leetcode.cn/problems/longest-palindromic-subsequence/">516.
+最长回文子序列</a></p>
+<p>回文，需要两个指针，二维DP，从i到j组成回文的长度，元素是长度，最小长度为1，相等+2</p>
+<p><a href="https://leetcode.cn/problems/ones-and-zeroes/">474.
+一和零</a></p>
+<p>字符串数组，组成数量限制（两个限制），所以限制需要二个维度，数组作为最外层那个维度，元素是个数，最小为0</p>
+<p><a
+href="https://leetcode.cn/problems/delete-operation-for-two-strings/">583.
+两个字符串的删除操作</a></p>
+<p>两个字符串，两个维度，删除次数，初始情况，删除个数为字符串长度。</p>
+<p><a href="https://leetcode.cn/problems/palindromic-substrings/">647.
+回文子串</a></p>
+<p>回文，二维DP，元素是bool判断，统计True个数</p>
+<p><a
+href="https://leetcode.cn/problems/longest-common-subsequence/">1143.
+最长公共子序列</a></p>
+<p>两个字符串，二维DP，长度</p>
+<p><a href="https://leetcode.cn/problems/jump-game-vii/">1871. 跳跃游戏
+VII</a></p>
+<p>能否到达终点，一维DP，反向思考，能否到达该点，所以元素为bool。+前缀和优化时间</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 回文串</span></span><br><span class="line">dp = [[<span class="literal">False</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">    dp[i][i] = <span class="literal">True</span></span><br><span class="line">    <span class="comment"># 从i到j是否组成回文串</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n-<span class="number">1</span>)[::-<span class="number">1</span>]:</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(i+<span class="number">1</span>, n):</span><br><span class="line">            <span class="keyword">if</span> s[i] == s[j] <span class="keyword">and</span> (j - i &lt;= <span class="number">2</span> <span class="keyword">or</span> dp[i+<span class="number">1</span>][j-<span class="number">1</span>]):</span><br><span class="line">                dp[i][j] = <span class="literal">True</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h2 id="双指针">双指针</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-repeating-character-replacement/">424.
+替换后的最长重复字符</a></p>
+<p>枚举右端点，如果没超过限制就继续移动右指针，否则移动左指针</p>
+<h2 id="栈">栈</h2>
+<p>PASS</p>
+<h2 id="字典树">字典树</h2>
+<p>PASS</p>
+<h2 id="模拟深搜">模拟/深搜</h2>
+<p>PASS</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>数学</tag>
+        <tag>字符串</tag>
       </tags>
   </entry>
   <entry>
-    <title>Mongodb：副本集配置</title>
-    <url>/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</url>
-    <content><![CDATA[<p>Mongodb4.2.x数据库配置的第二部分，副本集架构的配置</p>
+    <title>Hadoop伪分布式安装（Centos7）</title>
+    <url>/hexo_blog/2017/04/09/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Hadoop%E4%BC%AA%E5%88%86%E5%B8%83%E5%BC%8F%E5%AE%89%E8%A3%85%EF%BC%88Centos7%EF%BC%89/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Hadoop的伪分布式安装和一些基础操作</p>
 <span id="more"></span>
-<h2 id="准备">准备</h2>
-<p>接
-<a href="/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/" title="Mongodb从安装到基本操作">Mongodb从安装到基本操作</a>，基本同原文，由于是副本集配置，所以在需要配置的服务器上进行相同的配置</p>
-<p>选择：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/wKiom1SFSNnAeaEUAADD3ZcjTjw828.jpg" /></p>
+<h2 id="文章开头">文章开头</h2>
+<p>环境配置：Centos7</p>
+<p>Hadoop版本：hadoop：2.7.3</p>
+<p>JDK版本：jdk-8u111-linux-x64.tar.gz</p>
+<h2 id="安装之前">安装之前</h2>
 <ol type="1">
-<li>单机上，配置不同port的mongo服务</li>
-<li>不同机器上，配置mongo服务</li>
-<li>要求同网段，所以配置的ip中不能写127.0.0.1</li>
-<li>可以 <code>vim /etc/hosts</code>，方便管理</li>
+<li><p>下载hadoop安装包, <a
+href="http://hadoop.apache.org/releases.html">下载网址</a>。进入官网之后，点击对应版本的“binary”，之后点击链接下载即可。</p></li>
+<li><p>jdk。通过官网下载jdk</p></li>
+<li><p>通过Xshell，上传两个安装包至/usr/local/。服务器与本地下载和上传需要lrzsz这个软件</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载压缩包</span></span><br><span class="line">wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.15.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br></pre></td></tr></table></figure>
-<p>更新部分，配置文件用yaml格式</p>
-<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="attr">systemLog:</span></span><br><span class="line">  <span class="attr">destination:</span> <span class="string">file</span></span><br><span class="line">  <span class="attr">path:</span> <span class="string">&quot;/opt/mongodb/data/test/logs/mongodb.log&quot;</span></span><br><span class="line">  <span class="attr">logAppend:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">storage:</span></span><br><span class="line">  <span class="attr">dbPath:</span> <span class="string">&quot;/opt/mongodb/data/test/db&quot;</span></span><br><span class="line">  <span class="attr">journal:</span></span><br><span class="line">    <span class="attr">enabled:</span> <span class="literal">true</span> </span><br><span class="line"><span class="attr">net:</span></span><br><span class="line">  <span class="comment"># 允许所有IP访问，也可以用逗号隔开，控制IP访问</span></span><br><span class="line">  <span class="attr">bindIp:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span> </span><br><span class="line">  <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">  <span class="attr">port:</span> <span class="number">27017</span></span><br><span class="line"><span class="attr">processManagement:</span></span><br><span class="line">  <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">  <span class="attr">fork:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">replication:</span></span><br><span class="line">  <span class="comment"># 副本集名字</span></span><br><span class="line">  <span class="attr">replSetName:</span> <span class="string">&quot;rs0&quot;</span> </span><br></pre></td></tr></table></figure>
-<h3 id="启动">启动</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在所有机器上启动mongo，此时无账号密码</span></span><br><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
-<h3 id="配置副本集">配置副本集</h3>
-<p>在任意一台服务器A上，进行配置副本集</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将副本集的初始化配置写入js</span></span><br><span class="line">vim init_replica.js</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 其中rs0一定要跟上面的replSetName一样</span></span><br><span class="line">config = &#123; _id:<span class="string">&quot;rs0&quot;</span>, members:[&#123;_id:0,host:<span class="string">&quot;ip0:port0&quot;</span>&#125;,&#123;_id:1,host:<span class="string">&quot;ip1:port1&quot;</span>&#125;]&#125;</span><br><span class="line">rs.initiate(config) </span><br><span class="line">rs.status()</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/mongod --port 27017 &lt; init_replica.js</span><br></pre></td></tr></table></figure>
-<h3 id="验证">验证</h3>
-<p>在A服务器上进行操作，添加账号密码</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">bin/mongod --port 27017</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123; user: <span class="string">&quot;admin&quot;</span>, <span class="built_in">pwd</span>: <span class="string">&quot;password&quot;</span>, roles: [&#123; role: <span class="string">&quot;userAdminAnyDatabase&quot;</span>, db: <span class="string">&quot;admin&quot;</span> &#125;] &#125;)</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找到mongo的服务进程pid</span></span><br><span class="line">ps ux | grep mongod</span><br><span class="line"><span class="comment"># 杀死进程，填写对应的pid</span></span><br><span class="line"><span class="built_in">kill</span> PID</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">bin/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
-<h3 id="openssl证书文件">OpenSSL证书文件</h3>
-<h3 id="读写分离">读写分离</h3>
-<h3 id="主节点选举配置">主节点选举配置</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">yum install lrzsz</span><br></pre></td></tr></table></figure>
+<p>之后可手动拖动文件至服务器当前目录下，也可通过命令
+“rz”来打开窗口，从而进行上传。 （下载命令“sz”）</p>
+<h2 id="解压缩及配置">解压缩及配置</h2>
+<p>在<code>/usr/local/</code>目录下进行解压 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#请根据下载版本，进行解压缩</span></span><br><span class="line">tar -zxvf jdk-8u111-linux-x64.tar.gz</span><br><span class="line">tar -zxvf hadoop-2.7.3.tar.gz</span><br></pre></td></tr></table></figure></p>
+<h3
+id="配置java环境变量可与配置hadoop环境变量配置一起进行">配置Java环境变量（可与配置Hadoop环境变量配置一起进行）</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#第十行左右输入命令，注意版本</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
+<h3 id="配置hadoop环境变量">配置Hadoop环境变量</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#在刚刚JAVA_HOME之下一行，注意版本</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#此处的PATH承接上面Java环境变量的配置，即是在原来基础上进行增加而不是另起一行</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/usr/local/hadoop-2.7.3</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#更新设置</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
+<h3 id="配置启动hadoop">配置启动Hadoop</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="comment">#修改配置文件</span></span><br><span class="line">vim hadoop-env.sh</span><br><span class="line"><span class="comment">#第二十五行</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim core-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 指定HDFS老大（namenode）的通信地址 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">	    &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">	    &lt;value&gt;hdfs://hadoopbonc1:9000&lt;/value&gt;</span><br><span class="line">    &lt;!-- 指定hadoop运行时产生文件的存储路径 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#此处fs.defaultFS的vaule写本机主机名，即第一个value的hadoopbonc1换成主机名</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim hdfs-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 设置hdfs副本数量 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;1&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br></pre></td></tr></table></figure>
+<h3 id="配置ssh免密码登陆密码互通">配置SSH免密码登陆（密码互通）</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#配置前</span></span><br><span class="line">ssh localhost</span><br><span class="line"><span class="comment">#回车后，要求输入本机登陆密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#配置主机名</span></span><br><span class="line">vim /etc/sysconfig/network</span><br><span class="line"><span class="built_in">source</span> /etc/sysconfig/network</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置hosts</span></span><br><span class="line">vim /etc/hosts</span><br><span class="line">ssh-keygen</span><br><span class="line"></span><br><span class="line"><span class="comment">#把公钥和私钥复制到相应的节点（因为本机是单节点，所以复制到本机）</span></span><br><span class="line"><span class="comment">#此处的hadoopbonc1换成主机名</span></span><br><span class="line">ssh-copy-id hadoopbonc1</span><br></pre></td></tr></table></figure>
+<h3 id="hdfs启动与停止">hdfs启动与停止</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#第一次启动需要格式化，理由跟磁盘第一次使用需要格式化一样</span></span><br><span class="line"><span class="comment">#以后启动就不需要格式化</span></span><br><span class="line">hdfs namenode -format</span><br><span class="line"><span class="comment">#启动hdfs</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#在hadoop目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/locate/hadoop-2.7.3/</span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">./sbin/start-dfs.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现四行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:50070</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#若无响应则需要开放50070端口，则需开放50070端口</span></span><br><span class="line">firewall-cmd --zone=public --add-port=50070/tcp --permanent</span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment">#hdfs的停止</span></span><br><span class="line"><span class="comment">#同样在hadoop目录下</span></span><br><span class="line">./sbin/stop-dfs.sh</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="配置和启动yarn">配置和启动YARN</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#切换配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="built_in">mv</span> mapred-site.xml.template mapred-site.xml</span><br><span class="line">vim mapred-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 通知框架MR使用YARN --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;yarn&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim yarn-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- reducer取数据的方式是mapreduce_shuffle --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;mapreduce_shuffle&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#启动YARN</span></span><br><span class="line">start-yarn.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现六行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps、ResourceManager、NodeManager，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:8088</span></span><br><span class="line"><span class="comment">#若无响应，可参照上文中的开放50070端口，开放8088端口，只需将50070换为8088即可</span></span><br></pre></td></tr></table></figure>
+<h2 id="测试">测试</h2>
+<p>在本地新建一个文件，如在<code>/home/user/</code>下新建<code>words.txt</code>，内容如下
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hello world</span><br><span class="line">hello hadoop</span><br><span class="line">hello csdn</span><br><span class="line">hello</span><br></pre></td></tr></table></figure></p>
+<p>正式进行测试。命令如下： <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hdfs根目录下新建test目录</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">mkdir</span> /test</span><br><span class="line"></span><br><span class="line"><span class="comment">#查看hdfs根目录下的目录结构</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /</span><br><span class="line"></span><br><span class="line"><span class="comment">#将本地文件上传至/test/目录下</span></span><br><span class="line">bin/hdfs dfs -put /home/user/words.txt /test/</span><br><span class="line"></span><br><span class="line"><span class="comment">#运行wordcount</span></span><br><span class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /test/words.txt /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#在/test/目录下生成了一个名为out的文件目录，查看一下/out/目录下的文件</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#结果保存在part-r-00000，查看一下运行结果</span></span><br><span class="line">bin/hdfs fs -<span class="built_in">cat</span> /test/out/part-r-00000</span><br></pre></td></tr></table></figure></p>
+<h2 id="hdfs的常用操作命令">HDFS的常用操作命令</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#常用操作：</span></span><br><span class="line"><span class="comment">#HDFS shell</span></span><br><span class="line"><span class="comment">#查看帮助</span></span><br><span class="line">hadoop fs -<span class="built_in">help</span> &lt;cmd&gt;</span><br><span class="line"><span class="comment">#上传</span></span><br><span class="line">hadoop fs -<span class="built_in">cat</span> &lt;hdfs上的路径&gt;</span><br><span class="line"><span class="comment">#查看文件列表</span></span><br><span class="line">hadoop fs -<span class="built_in">ls</span> /</span><br><span class="line"><span class="comment">#下载文件</span></span><br><span class="line">hadoop fs -get &lt;hdfs上的路径&gt; &lt;linux上文件&gt;</span><br></pre></td></tr></table></figure>
+<p>下一篇：<a href="/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/" title="在Hadoop基础上Hive的安装">在Hadoop基础上Hive的安装</a></p>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Environment</category>
       </categories>
       <tags>
-        <tag>mongodb</tag>
-        <tag>数据库</tag>
+        <tag>Linux</tag>
+        <tag>Hadoop</tag>
+        <tag>大数据</tag>
       </tags>
   </entry>
   <entry>
-    <title>Mongodb从安装到基本操作</title>
-    <url>/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</url>
-    <content><![CDATA[<p>这篇文章主要从零开始介绍如何在Centos7下安装Mongodb,
-用Python3连接Mongodb, 用可视化工具Robo 3T连接Mongodb,
-之后介绍了一些基本的mongo操作方法, 其中含有一些进阶操作</p>
+    <title>在Hadoop基础上Hive的安装</title>
+    <url>/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Hive在Hadoop上的安装及一些基本的Hive操作</p>
 <span id="more"></span>
-<h2 id="安装mongodbcentos7">安装Mongodb(Centos7)</h2>
+<p>写在开头 环境 Hadoop单机 Centos7 Hadoop-2.7.3
+hadoop位置：/usr/loacl/hadoop</p>
+<p>参考文章 <a
+href="http://yanliu.org/2015/08/13/Hadoop%E9%9B%86%E7%BE%A4%E4%B9%8BHive%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">Hadoop集群之Hive安装配置</a></p>
+<h2 id="下载hive">下载Hive</h2>
+<p>下载源码包 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hadoop目录下操作</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop</span><br><span class="line"></span><br><span class="line"><span class="comment">#用wget下载</span></span><br><span class="line">wget http://mirrors.cnnic.cn/apache/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#也可以在图形界面下下载之后，上传压缩包</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#解压缩包</span></span><br><span class="line">tar -zxvf apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置环境变量</span></span><br><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#位置在之前配置的变量之后，大概12行左右，因为之前配置了jdk和hadoop变量</span></span><br><span class="line"><span class="built_in">export</span> HIVE_HOME=/usr/local/hadoop/apache-hive-1.2.2-bin</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HIVE_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#适当位置添加&quot;$HIVE_HOME/bin:&quot;</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#使文件生效</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure></p>
+<h2 id="安装mysql">安装MySQL</h2>
+<p>MariaDB数据库管理系统是MySQL的一个分支，主要由开源社区在维护，采用GPL授权许可。开发这个分支的原因之一是：甲骨文公司收购了MySQL后，有将MySQL闭源的潜在风险，因此社区采用分支的方式来避开这个风险。MariaDB的目的是完全兼容MySQL，包括API和命令行，使之能轻松成为MySQL的代替品。</p>
+<p>在Centos7下，使用命令安装mysql，会安装成mariadb。</p>
+<p>在这里先介绍安装MariaDB，原因如上。</p>
+<h3 id="安装mariadb">安装MariaDB</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#安装</span></span><br><span class="line">yum install mariadb-server mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">systemctl start mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入MySQL</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下，可能有点不习惯</span></span><br><span class="line">MariaDB [(none)]&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#P.S.</span></span><br><span class="line"><span class="comment">#相关命令</span></span><br><span class="line">systemctl start mariadb <span class="comment">#启动MariaDB</span></span><br><span class="line">systemctl stop mariadb <span class="comment">#停止MariaDB</span></span><br><span class="line">systemctl restart mariadb <span class="comment">#重启MariaDB</span></span><br><span class="line">systemctl <span class="built_in">enable</span> mariadb <span class="comment">#设置开机启动</span></span><br></pre></td></tr></table></figure>
+<h3 id="正式安装mysql">正式安装MySQL</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下载，在这里使用的是命令行下载，也建议在图形界面下载，然后上传至服务器</span></span><br><span class="line">wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用rpm安装</span></span><br><span class="line">rpm -ivh mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用yum安装mysql-community-server</span></span><br><span class="line">yum install mysql-community-server</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动服务</span></span><br><span class="line">service mysqld start</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入mysql，第一次进入无密码</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下</span></span><br><span class="line">mysql&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="配置mysql">配置MySQL</h3>
+<p>编码配置 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件，若未安装vim，建议先使用命令yum install vim安装vim</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"></span><br><span class="line"><span class="comment">#最后加上编码配置</span></span><br><span class="line">[mysql]</span><br><span class="line">default-character-set =utf8</span><br><span class="line"></span><br><span class="line"><span class="comment">#此处字符编码必须和/usr/share/mysql/charsets/Index.xml中一致。</span></span><br><span class="line"><span class="comment">#不过一般情况下使用的都是utf8</span></span><br></pre></td></tr></table></figure></p>
+<p>设置密码 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下面三种方法需要进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#方法一</span></span><br><span class="line">mysql&gt;insert into user(host,user,password) values(<span class="string">&#x27;%&#x27;</span>,<span class="string">&#x27;user_name&#x27;</span>,password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法二</span></span><br><span class="line">mysql&gt;<span class="built_in">set</span> password <span class="keyword">for</span> user_name = password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法三</span></span><br><span class="line">mysql&gt;grant all on *.* to user_name@% identified by <span class="string">&quot;password&quot;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">#下面这一种方法可直接在shell下设置密码</span></span><br><span class="line">mysqladmin -u root password <span class="string">&quot;password&quot;</span></span><br></pre></td></tr></table></figure> 远程连接</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#把在所有数据库的所有表的所有权限赋值给位于所有IP地址的root用户。</span></span><br><span class="line">mysql&gt; grant all privileges on *.* to root@<span class="string">&#x27;%&#x27;</span>identified by <span class="string">&quot;password&quot;</span>;</span><br></pre></td></tr></table></figure>
+<p>P.S.上文引号中的user_name表示数据库的用户名，password表示对应用户的密码。即这两项是由读者自行定义的。</p>
+<h3 id="常见问题及解决方案">常见问题及解决方案</h3>
+<h4 id="mysql创建用户失败">Mysql创建用户失败</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># mysql在my.ini的配置文件中设置了严格模式，所以我们需要进行修改</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第一步，寻找配置文件</span></span><br><span class="line">whereis my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步，根据上一步结果进行vim编辑</span></span><br><span class="line">vim /../my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第三步，在vim里面搜索sql-mode,删除STRICT_TRANS_TABLES，保存退出即可</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第四步，保险起见，使刚刚的配置文件立即生效</span></span><br><span class="line"><span class="built_in">source</span> /../my.ini</span><br></pre></td></tr></table></figure>
+<h4 id="支持中文">支持中文</h4>
+<ul>
+<li><p>Centos7</p>
+<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 编辑文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加修改如下代码</span></span><br><span class="line">[client]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-character-set=utf8</span><br><span class="line"></span><br><span class="line">[mysqld]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-storage-engine=INNODB</span><br><span class="line">character-set-server=utf8</span><br><span class="line">collation-server=utf8_general_ci</span><br><span class="line"></span><br><span class="line">[mysql]</span><br><span class="line">no-auto-rehash</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 保存退出，重启服务</span></span><br><span class="line"><span class="comment"># 重新登陆mysql检查是否成功，方法见下</span></span><br></pre></td></tr></table></figure></p></li>
+<li><p>Ubuntu16.04</p>
+<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> vim /etc/mysql/mysql.conf.d/mysqld.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加以下内容，如果不存在[xx]自行增加</span></span><br><span class="line">[mysqld]</span><br><span class="line">character_set_server=utf8</span><br><span class="line">[mysql]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysql.server]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysqld_safe]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[client]</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 重启mysql</span></span><br><span class="line">service mysql restart</span><br><span class="line"><span class="comment"># 进入mysql查看配置参数</span></span><br><span class="line">mysql -uroot -p</span><br><span class="line"><span class="comment"># 查看database 的value变为utf8即可</span></span><br><span class="line">&gt;show variables like <span class="string">&#x27;%character%&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment"># p.s.如果之前创建了表，表编码不会改变</span></span><br></pre></td></tr></table></figure></p></li>
+</ul>
+<h4
+id="mysql报错1366---incorrect-integer-value-xxxxxxx-at-row-1">MySQL报错“1366
+- Incorrect integer value: '' XXXXXXX' at row 1 ”</h4>
+<p>修改方法:（两种,建议第二种）</p>
 <ol type="1">
-<li>官网上找到需要下载的版本压缩包进行下载，<a
-href="https://fastdl.mongodb.org/linux/">官网链接</a>。或者在命令行下使用<code>wget</code>命令进行下载</li>
+<li><p>命令行。set names gbk；(此为设置通信编码)</p></li>
+<li><p><code>my.cnf</code> 中查找sql-mode</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 这里下载的是3.4.10版本</span></span><br><span class="line">wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel70-3.4.10.tgz</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li>进行解压缩</li>
+<p>将 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 修改为 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 重启mysql后即可</p>
+<h4 id="关于my.cnf和my.ini的说明">关于my.cnf和my.ini的说明</h4>
+<p>my.cnf常见于Linux系统，my.ini常见与windows系统，二者都是属于mysql的配置文件。一般好像在一个系统下就是只出现一种配置文件，具体区别没有深入了解，修改配置文件，根据自己的系统进行查找修改配置文件即可</p>
+<h4 id="mysql交互环境自动补全">Mysql交互环境自动补全</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 修改配置文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在[mysql]部分添加auto-rehash</span></span><br><span class="line"><span class="comment"># 保存退出，重启</span></span><br><span class="line">service mysqld restart</span><br></pre></td></tr></table></figure>
+<h2 id="配置hive">配置Hive</h2>
+<p>我们之前在hadoop目录下安装了Hive，位置为<code>/usr/local/hadoop/apache-hive-1.2.1-bin</code></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入hive配置目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/conf</span><br><span class="line"></span><br><span class="line"><span class="comment">#修改hive-default.xml.template</span></span><br><span class="line"><span class="comment">#首先复制</span></span><br><span class="line"><span class="built_in">cp</span> hive-default.xml.template hive-default.xml</span><br><span class="line"><span class="comment">#修改文件</span></span><br><span class="line">vim hive-default.xml</span><br><span class="line"></span><br><span class="line"><span class="comment"># 1. 第一步将&lt;configuration&gt;&lt;/configuration&gt;中内容删除大概是18-3908行</span></span><br><span class="line"><span class="comment"># vim删除命令--&gt;:18,3908d</span></span><br><span class="line"><span class="comment"># 2.将下面&lt;configuration&gt;&lt;/configuration&gt;中内容复制进去,分别将3306喝9083前的user_name改为当前的用户名</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionURL&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;jdbc:mysql://user_name:3306/hive?createDatabaseIfNotExist=<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;JDBC connect string <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionDriverName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;com.mysql.jdbc.Driver&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;Driver class name <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionUserName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;value&gt;</span><br><span class="line">        &lt;description&gt;username to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionPassword&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;password to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">    &lt;name&gt;hive.metastore.uris&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;thrift://user_name:9083&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
+<h2 id="下载jdbc">下载JDBC</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#命令行下载,也可以使用图形界面上传文件</span></span><br><span class="line"> wget http://cdn.mysql.com/Downloads/Connector-J/mysql-connector-java-5.1.36.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#将文件复制进Hive的lib目录下，原因，Hive的自带的那个版本低，可能失效</span></span><br><span class="line"> <span class="built_in">cp</span> mysql-connector-java-5.1.33-bin.jar /usr/local/hadoop/apache-hive-1.2.1-bin/lib/</span><br></pre></td></tr></table></figure>
+<p>启动Hive <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hive --service metastore &amp;</span><br><span class="line">jps</span><br><span class="line"><span class="comment">#结果会多出一个进程</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#进入hive目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/bin</span><br><span class="line"><span class="comment">#启动hive，可能有点慢</span></span><br><span class="line">hive</span><br><span class="line"></span><br><span class="line"><span class="comment">#若出现hive的命令行即代表成功，如下</span></span><br><span class="line">hive&gt;</span><br></pre></td></tr></table></figure></p>
+<h2 id="常见问题及解决方案-1">常见问题及解决方案</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">Logging initialized using configuration <span class="keyword">in</span> jar:file:/home/hadoop/apache-hive-1.2.1-bin/lib/hive-common-1.2.1.jar!/hive-log4j.properties</span><br><span class="line">Exception <span class="keyword">in</span> thread <span class="string">&quot;main&quot;</span> java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient</span><br><span class="line">        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)</span><br></pre></td></tr></table></figure>
+<p>解决方法，直接关闭防火墙</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#这里的系统为Centos7，所以使用此命令</span></span><br><span class="line">systemctl stop firewalld</span><br></pre></td></tr></table></figure>
+<p>hive metastore 启动出错解决 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看与hive相关进程是否启动</span></span><br><span class="line">ps -ef | grep hive</span><br><span class="line"><span class="comment"># kill相关进程，为进程号</span></span><br><span class="line"><span class="built_in">kill</span> num</span><br><span class="line"><span class="comment"># 重新启动</span></span><br><span class="line">./hive</span><br></pre></td></tr></table></figure></p>
+<p>更多问题见此文章 <a
+href="http://blog.csdn.net/freedomboy319/article/details/44828337">Hive常见问题汇总</a></p>
+<h2 id="hive的学习笔记">Hive的学习笔记</h2>
+<h3 id="新建表">新建表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 新建一张表，名为“test”，里面有“name”、“id”两类，分别是“string”、“int”的数据类型，以“|”隔开一列，表是作为textfile的。</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">CREATE</span> <span class="keyword">TABLE</span> test (name String, id <span class="type">int</span>) <span class="type">ROW</span> FORMAT DELIMITED FIELDS TERMINATED <span class="keyword">BY</span> <span class="string">&#x27;|&#x27;</span> STORED <span class="keyword">AS</span> TEXTFILE;</span><br></pre></td></tr></table></figure>
+<h3 id="加载表">加载表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 从本地的/home/user/test.txt文件，将数据加载进test这个表</span></span><br><span class="line">hive<span class="operator">&gt;</span> LOAD DATA <span class="keyword">LOCAL</span> INPATH <span class="string">&#x27;/home/user/test.txt&#x27;</span> OVERWRITE <span class="keyword">INTO</span> <span class="keyword">TABLE</span> test;</span><br></pre></td></tr></table></figure>
+<h3 id="关联表">关联表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 将两张表通过一个或多个字段关联在一起</span></span><br><span class="line"><span class="comment">-- 将test_a和test_b表的id字段关联在一起，再加上“name”字段，组成一个新表</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> test_a.id, test_b.name <span class="keyword">FROM</span> test_a, test_b <span class="keyword">JOIN</span> test_b <span class="keyword">ON</span> (test_a.id <span class="operator">=</span> test_b.id);</span><br></pre></td></tr></table></figure>
+<h3 id="保存表">保存表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 由于hive下执行任务之后，并不会保存数据，所以我们使用INSERT命令来保存命令</span></span><br><span class="line"><span class="comment">-- 插入数据到test表中</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">TABLE</span> test</span><br><span class="line">     <span class="operator">&gt;</span> ...</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 保存表到hdfs上,保存输出结果到hdfs下的/out/目录下</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">LOCAL</span> DIRECTORY &quot;/out/&quot;</span><br><span class="line">    <span class="operator">&gt;</span> ...</span><br></pre></td></tr></table></figure>
+<h3 id="排序问题">排序问题</h3>
+<ol type="1">
+<li>order by</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br></pre></td></tr></table></figure>
-<ol start="3" type="1">
-<li>进行相应的配置，mongodb默认没有任何配置</li>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li><p>sort by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test SORT <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
+<li><p>distribute by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> name DISTRIBUTE <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
+<li><p>DISTRIBUTE BY with SORT BY DISTRIBUTE BY 和 GROUP BY
+有点类似，DISTRIBUTE BY控制reduce如何处理数据，而SORT
+BY控制reduce中的数据如何排序。 注意：hive要求DISTRIBUTE BY语句出现在SORT
+BY语句之前。</p></li>
+<li><p>Cluster By</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br><span class="line">  `</span><br><span class="line">    <span class="comment"># 设置数据文件的存放目录</span></span><br><span class="line">    dbpath = /opt/mongodb/data/test/db</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置日志文件的存放目录及其日志文件名</span></span><br><span class="line">    logpath = /opt/mongodb/data/test/logs/mongodb.log</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">    port = 27017</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">    fork = <span class="literal">true</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 是否不允许表扫描</span></span><br><span class="line">    nohttpinterface = <span class="literal">true</span></span><br><span class="line">  `</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
-<ol start="4" type="1">
-<li>启动mongodb</li>
+<p>cluster by 除了具有 distribute by 的功能外还兼具 sort by 的功能。
+默认升序排序，但DISTRIBUTE BY的字段和SORT
+BY的字段必须相同，且不能指定排序规则。</p>
+<p>总结：</p>
+<ul>
+<li><code>ORDER BY</code>是全局排序，但在数据量大的情况下，花费时间会很长</li>
+<li><code>SORT BY</code>是将<code>reduce</code>的单个输出进行排序，不能保证全局有序</li>
+<li><code>DISTRIBUTE BY</code>
+可以按指定字段将数据划分到不同的<code>reduce</code>中</li>
+<li>当<code>DISTRIBUTE BY</code> 的字段和
+<code>SORT BY</code>的字段相同时，可以用<code>CLUSTER BY</code>来代替<code>DISTRIBUTE BY with SORT BY</code>。</li>
+</ul>
+<h3 id="hive常用函数">hive常用函数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以下无特殊说明，返回值皆为string</span></span><br><span class="line"><span class="comment"># 字符串连接</span></span><br><span class="line">hive&gt;concat(strA, strB, ...)</span><br><span class="line"><span class="comment"># 带分隔符的字符串连接，即连接之后字符串存在分隔符，strSEP表示分隔符</span></span><br><span class="line">hive&gt;concat_wd(strSEP, strA, strB, ...)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串截取</span></span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到结尾为止</span></span><br><span class="line">hive&gt;substr(strA, num0)</span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到num1为止</span></span><br><span class="line">hive&gt;substr(strA, num0, num1)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串转大写</span></span><br><span class="line">hive&gt;upper(str)</span><br><span class="line"><span class="comment"># 字符串转小写</span></span><br><span class="line">hive&gt;lower(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 除去字符串中空格</span></span><br><span class="line">hive&gt;trim(str)</span><br><span class="line"><span class="comment"># 除去字符串左边空格</span></span><br><span class="line">hive&gt;ltrim(str)</span><br><span class="line"><span class="comment"># 除去字符串右边空格</span></span><br><span class="line">hive&gt;rtrim(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则替换字符</span></span><br><span class="line"><span class="comment"># 将strA中符合strRE的部分替换为strB</span></span><br><span class="line">hive&gt;regexp_replace(strA, strRE, strB)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重复字符串</span></span><br><span class="line"><span class="comment"># str为字符串，num为重复次数</span></span><br><span class="line">hive&gt;repeat(str, num)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 分割字符串</span></span><br><span class="line"><span class="comment"># 以strpat为分割符，返回字符串数组</span></span><br><span class="line">hive&gt;spilt(str, strpat)</span><br></pre></td></tr></table></figure>
+<h3 id="hive导出数据">hive导出数据</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导出至本地，content表示本地目录</span></span><br><span class="line">hive&gt;insert overwrite <span class="built_in">local</span> directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hdfs，content表示hdfs目录</span></span><br><span class="line">hive&gt;insert overwrite directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hive另一张表othertable，前提hive中存在这张表</span></span><br><span class="line">hive&gt;insert overwrite othertable</span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br></pre></td></tr></table></figure>
+<h3 id="转换类型">转换类型</h3>
+<p>cast(xxx as xx) 将xxx类型转换为xx类型</p>
+<p>下表为是否可转换类型的说明 <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/20170612195815151.png" /></p>
+<p>注：由于表格比较大，这里对一些比较长的字符串进行缩写，ts是timestamp的缩写,bl是boolean的缩写,sl是smallint的缩写,dm是decimal的缩写,vc是varchar的缩写,ba是binary的缩写。</p>
+]]></content>
+      <categories>
+        <category>Environment</category>
+      </categories>
+      <tags>
+        <tag>Hadoop</tag>
+        <tag>大数据</tag>
+        <tag>Hive</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>最优化算法学习记录</title>
+    <url>/hexo_blog/2021/02/03/%E6%95%B0%E5%AD%A6/%E6%9C%80%E4%BC%98%E5%8C%96%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>本文提供了优化问题的概述，包括背景，分类，以及线性，非线性和二次规划问题的求解方法。本文还讨论了全局优化，并提供了非凸函数和凸函数的例子，以及线性规划和二次规划的例子。</p>
+<span id="more"></span>
+<p>优化问题的最终目的就是求最小值（最大值转为求最小值）</p>
+<h3 id="简单划分">简单划分</h3>
+<ol type="1">
+<li>解析法</li>
+<li><strong>迭代法</strong>
+<ol type="1">
+<li>常规方法</li>
+<li>技巧
+<ol type="1">
+<li>切平面</li>
+<li>椭圆法</li>
+<li>内点法</li>
+</ol></li>
+</ol></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">./bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
-<p>报错一：
-<code>ERROR: child process failed, exited with error number 1</code>
-检查mongodb.conf的文件路径是否配置错误</p>
-<p>报错二：<code>ERROR: child process failed, exited with error number 100</code>
-很可能是没有正常关闭导致的，那么可以删除
-<code>mongod.lock</code>文件，这里对应我的配置路径在<code>data/</code>里面</p>
-<ol start="5" type="1">
-<li>其他</li>
+<h3 id="background">Background</h3>
+<ol type="1">
+<li>目标函数、约束、线性（非线性）</li>
+<li>凸优化问题（是否为凸问题）</li>
+<li>局部最优、全局最优</li>
+<li>矩阵求导（梯度），矩阵的一阶导数、二阶导数（Hessian）</li>
+<li>解析与迭代</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 链接命令，方便调用mongo命令</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/mongodb/bin/mongo /usr/bin</span><br><span class="line"><span class="comment"># 查看mongodb进程</span></span><br><span class="line">ps -aux |grep mongodb</span><br><span class="line"><span class="comment"># 检查端口运行情况</span></span><br><span class="line">netstat -lanp | grep 27017</span><br><span class="line"><span class="comment"># 终止mongodb服务，PID从ps命令获取</span></span><br><span class="line"><span class="built_in">kill</span> -15 PID</span><br><span class="line"><span class="comment"># 添加自启动命令</span></span><br><span class="line">vim /etc/rc.local</span><br><span class="line"><span class="comment"># 末尾追加一行</span></span><br><span class="line">`/opt/mongodb/bin/mongod --config mongodb.conf`</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
-<ol start="6" type="1">
-<li>设置密码权限，默认无密码</li>
+<h3 id="how">How</h3>
+<ol type="1">
+<li>目标函数、约束均为线性：单纯形法。大问题：内点法</li>
+<li>目标函数二次，约束线性：改进单纯形法。大问题：内点法</li>
+<li>凸问题：切割平面、椭球算法、<strong>内点法</strong></li>
+<li>目标函数存在多个局部最小值，全局优化算法（启发式算法/智能优化算法）</li>
+<li>非线性非凸问题（最难），梯度和海森信息（二阶导）-&gt;构造解析表达式</li>
+<li>无约束问题
+<ol type="1">
+<li>Levenberg-Marquardt</li>
+<li>牛顿算法</li>
+<li>拟牛顿</li>
+<li>最速下降</li>
+<li>Powell</li>
+</ol></li>
+<li>线性约束：SQP、梯度投影法</li>
+<li>非线性约束：SQP、转为为无约束</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">./mongo</span><br><span class="line"><span class="comment"># 如果进入失败，请检查是否添加了软链接和是否启动了mongodb服务</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"><span class="comment"># 重启mongodb服务</span></span><br><span class="line"><span class="comment"># 杀死mongodb进程，参照上面的方法</span></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">opt/mongodb/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
-<h2 id="使用python连接mongodb">使用Python连接Mongodb</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载第三方包</span></span><br><span class="line">pip3 install pymongo</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># /usr/bin/python3</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="comment"># host主机名，27017连接端口</span></span><br><span class="line">client = MongoClient(host, <span class="number">27017</span>)</span><br><span class="line">db_auth = client.admin</span><br><span class="line"><span class="comment"># 登陆的用户名(username)和密码(password)</span></span><br><span class="line">db_auth.authenticate(username, password)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line"><span class="comment"># 用户名(username)、密码(password)、主机名(host)、端口(port)。注：这里的password不能出现@符号，如果用@符号就需要使用方法一</span></span><br><span class="line">client = MongoClient(<span class="string">&quot;mongodb://username:password@host:port&quot;</span>)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 操作数据库，db_name[&quot;db_set&quot;]=db_name.db_set</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">查找数据</span></span><br><span class="line"><span class="string">在db_name数据库中的db_set集合里面找到name是a_name的json</span></span><br><span class="line"><span class="string">find是返回一个指针（可以使用列表的方式来读取），find_one返回一个dict</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 插入数据</span></span><br><span class="line"><span class="comment"># 插入一个json到set1集合中，a_dict是一个字典（如果set1不存在则会被自动创建）</span></span><br><span class="line">db_name.set1.insert_one(a_dict)</span><br><span class="line"><span class="comment"># 插入多个json到set1集合中，a_dict_lst是一个列表，里面的元素是由字典组成的</span></span><br><span class="line">db_name.set1.insert_many(a_dict_lst)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更新数据</span></span><br><span class="line"><span class="comment"># 查找到一个数据之后，对里面的某个数据进行更改，再使用save方法保存</span></span><br><span class="line">data_one = db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line">data_one[<span class="string">&#x27;age&#x27;</span>] += <span class="number">3</span></span><br><span class="line">db_name.db_set.save(data_one)</span><br><span class="line"><span class="comment"># 更新多条记录，找到db_set集合中满足age=20，sex=0的数据，将name改为user1</span></span><br><span class="line">db_name.db_set.update(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;, &#123;<span class="string">&quot;$set&quot;</span>:&#123;<span class="string">&quot;age&quot;</span>: <span class="number">20</span>, <span class="string">&quot;sex&quot;</span>: <span class="number">0</span>&#125;&#125;)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除数据</span></span><br><span class="line"><span class="built_in">id</span> = db_name.db_set.find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)[<span class="string">&quot;_id&quot;</span>]</span><br><span class="line"><span class="comment"># 根据 id 删除一条记录</span></span><br><span class="line">db_name.db_set.remove(<span class="built_in">id</span>)</span><br><span class="line"><span class="comment"># 删除集合里的所有记录</span></span><br><span class="line">db_name.db_set.remove()</span><br><span class="line"><span class="comment"># 删除name=user1的记录</span></span><br><span class="line">db_name.db_set.remove(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)</span><br></pre></td></tr></table></figure>
-<h2 id="使用robo-3t连接mongodb">使用Robo 3t连接Mongodb</h2>
-<p>强烈推荐Robo 3t，<a href="https://robomongo.org/">官网地址</a></p>
+<h3 id="优化问题分类">优化问题分类</h3>
 <ol type="1">
-<li><p>到官网找到对应的系统版本进行下载</p></li>
-<li><p>安装</p>
-<ul>
-<li><p>Windows的安装十分简单就不赘述了</p></li>
-<li><p>Ubuntu的安装 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxzf robomongo*.tar.gz</span><br><span class="line"><span class="comment"># 修改文件名（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robomongo* robo3t</span><br><span class="line"><span class="comment"># 移动到/opt目录（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robo3t /opt</span><br><span class="line"><span class="comment"># 建立软链接</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/robo3t/robomongo /usr/bin</span><br><span class="line"><span class="comment"># 启动</span></span><br><span class="line">robomongo</span><br></pre></td></tr></table></figure> <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 如果报如下错误</span></span><br><span class="line">`</span><br><span class="line">This application failed to start because it could not find or load the Qt platform plugin <span class="string">&quot;xcb&quot;</span> <span class="keyword">in</span> <span class="string">&quot;&quot;</span>.</span><br><span class="line"></span><br><span class="line">Available platform plugins are: xcb.</span><br><span class="line"></span><br><span class="line">Reinstalling the application may fix this problem.</span><br><span class="line">Aborted (core dumped)</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 解决办法</span></span><br><span class="line"><span class="built_in">mkdir</span> ~/robo-backup</span><br><span class="line"><span class="built_in">mv</span> /opt/robo3t/lib/libstdc++* ~/robo-backup/</span><br><span class="line"><span class="comment"># 之后便可以重新启动mongo试试看了</span></span><br></pre></td></tr></table></figure></p></li>
-</ul></li>
-<li><p>使用</p>
+<li><p>无约束问题（非线性最小二乘问题）</p>
 <ol type="1">
-<li>首先创建新的连接，如下图所示。在<code>Connection中，``Name</code>为自定义连接名，<code>Address</code>填入对应的<code>host</code>和<code>port</code>；
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171027733.png" /></li>
-<li>在<code>Authentication</code>中，勾选<code>Perform authentication</code>，填上对应的<code>User Name</code>和<code>Password</code>。填写完成之后可以点击<code>Test</code>验证是否能够连接成功，如果无误的话，可以点击<code>Save</code>保存退出。</li>
+<li>牛顿法</li>
+<li>逆牛顿法</li>
+<li>Levenberg-Marquardt算法</li>
+<li>Nelder-Mead 方法</li>
+<li>其他
+<ol type="1">
+<li>搜索方向和一维搜索法</li>
+</ol></li>
+</ol></li>
+<li><p>约束问题</p>
+<ol type="1">
+<li>规划问题
+<ol type="1">
+<li>线性规划（整数规划）
+<ol type="1">
+<li>单纯形解法</li>
+</ol></li>
+<li>二次规划
+<ol type="1">
+<li>改进单纯形法</li>
+</ol></li>
+</ol></li>
+<li>非线性优化的约束
+<ol type="1">
+<li>等式约束
+<ol type="1">
+<li>线性等式约束</li>
+<li>非线性等式约束</li>
+</ol></li>
+<li>不等式约束
+<ol type="1">
+<li>线性等式约束</li>
+<li>非线性等式约束</li>
+</ol></li>
+</ol></li>
+</ol></li>
 </ol>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171111807.png" /></p>
-<ol start="3" type="1">
-<li>连接Mongodb。点击<code>Connect</code>，之后就可以可视化数据库中的数据。也可以使用鼠标简单的进行创建/删除数据库，新建/修改信息，也可以使用命令查找数据。</li>
+<h3 id="全局优化">全局优化</h3>
+<ol type="1">
+<li>启发式算法
+<ol type="1">
+<li>模拟退火</li>
+<li>遗传算法</li>
+</ol></li>
+<li>基本算法
+<ol type="1">
+<li>随机搜索</li>
+<li>多起点优化</li>
 </ol></li>
 </ol>
-<h2 id="curd操作主要是查找数据">CURD操作（主要是查找数据）</h2>
-<p><strong>CRUD</strong></p>
+<h3 id="优化问题fx">优化问题<span
+class="math inline">\(F(x)\)</span></h3>
 <ul>
-<li>C: Create Operations, 创建</li>
-<li>R: Read Operations, 读取</li>
-<li>U: Update Operations, 更新</li>
-<li>D: Delete Operations, 删除</li>
+<li>线性问题
+<ul>
+<li>不论有无约束，都可以使用<strong>单纯形法</strong>和<strong>内点法</strong>求解</li>
+</ul></li>
+<li>非线性问题
+<ul>
+<li>凸优化问题
+<ul>
+<li>有约束：切平面法、椭球法、内点法</li>
+<li>无约束情况下等同于非凸问题</li>
+</ul></li>
+<li>非凸问题
+<ul>
+<li>约束<span
+class="math inline">\(h(x)\)</span>不是线性，拉格朗日乘子法</li>
+<li>约束<span class="math inline">\(g(x)\)</span>为线性，gradient
+projection、SQP</li>
+<li>约束<span
+class="math inline">\(g(x)\)</span>不是线性，惩罚函数、Barrier函数、SQP问题</li>
+<li>无约束（可消约束）
+<ul>
+<li>多起点优化
+<ul>
+<li>Hession矩阵，一阶矩阵：Levenberg-Marquardt、牛顿法、牛顿线性搜索</li>
+<li>一阶矩阵：Quasi-Newton、Quasi-Newton线性搜索、最速下降法</li>
+<li>无：Powell Perpendicular线性搜索、Nelder-Mead</li>
+</ul></li>
+<li>不是多起点优化：随机搜索、模拟退火、遗传算法</li>
+</ul></li>
+</ul></li>
+</ul></li>
+<li>二次问题
+<ul>
+<li>约束，线性，修正的单纯形法、内点法</li>
+</ul></li>
 </ul>
-<p>mongo数据库至上而下是：<code>Database</code>--&gt;<code>Collections</code>--&gt;<code>Documents</code>。我在这里分别理解成数据库、集合、记录。</p>
-<p>这里直接用官方给的数据来操作。假设以下数据放在了<code>fruits</code>集合中。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">-1</span><span class="punctuation">,</span> <span class="number">3</span><span class="punctuation">]</span><span class="punctuation">,</span> <span class="attr">&quot;links&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span><span class="attr">&quot;Uin&quot;</span><span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span> <span class="attr">&quot;NickName&quot;</span><span class="punctuation">:</span> <span class="string">&quot;123&quot;</span><span class="punctuation">&#125;</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">1</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;oranges&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="attr">&quot;in stock&quot;</span><span class="punctuation">:</span> <span class="number">8</span><span class="punctuation">,</span> <span class="attr">&quot;ordered&quot;</span><span class="punctuation">:</span> <span class="number">12</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span>  <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span> <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;avocados&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="string">&quot;fourteen&quot;</span> <span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h3 id="基本操作">基本操作</h3>
-<p><code>json</code>数据的嵌套查找，就是说，如果有字段包含json，可以直接用<code>.</code>进行连接。</p>
-<p>查找<code>qty</code>中的<code>ordered</code>为12的字段</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty.ordered&quot;: <span class="number">12</span>&#125;)</span><br></pre></td></tr></table></figure>
-<p>多重筛选，同时满足多个要求的记录</p>
-<p>查找<code>qty</code>为5，且<code>_id</code>为"apples"的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>, &quot;_id&quot;: &quot;apples&quot;&#125;)</span><br></pre></td></tr></table></figure>
-<p>第一个大括号内容表示筛选条件，第二个大括号表示显示内容</p>
-<p>筛选条件就不说明了，第二个大括号表示打印<code>_id</code>字段和<code>score</code>字段，不打印其他字段，</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123; &quot;_id&quot;: <span class="number">1</span> , &quot;score&quot;: <span class="number">1</span>&#125;)</span><br></pre></td></tr></table></figure>
-<h3 id="比较">比较</h3>
-<p>下面至上而下分别表示，在<code>fruits</code>集合中查找字段<code>qty</code>与4的大小关系所有记录。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">4</span>&#125;)          # 等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">4</span>&#125;&#125;)   # 大于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lt&quot;: <span class="number">4</span>&#125;&#125;)   # 小于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gte&quot;: <span class="number">4</span>&#125;&#125;)  # 大于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lte&quot;: <span class="number">4</span>&#125;&#125;)  # 小于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$ne&quot;: <span class="number">4</span>&#125;&#125;)   # 不等于</span><br></pre></td></tr></table></figure>
-<p>当然，也可以两个比较符一起操作，比如下面这样。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">3</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
-<p>如果对<code>score</code>这个字段使用比较，比如下面这样。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;score&quot;: &#123;&quot;$gt&quot;: <span class="number">2</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
-<p>将会返回下面两条记录。理由是，字段里的数组里有含有大于2小于5的数字。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">3</span></span><br><span class="line">    <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span></span><br><span class="line">            <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;123&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">/* 2 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">5</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h3 id="关系">关系</h3>
-<h4 id="in">$in</h4>
-<p>查找<code>_id</code>字段中包含<code>5</code>或者<code>ObjectId("507c35dd8fada716c89d0013")</code>的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$in&quot;:[<span class="number">5</span>, ObjectId(&quot;507c35dd8fada716c89d0013&quot;)] &#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="elemmatch">$elemMatch</h4>
-<p>用于字段中含有数组，且数组中含有<code>json</code>数据的的查找。</p>
-<p>查找<code>links</code>这个含有数组的字段，找出数组中<code>Uin</code>等于124的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;links&quot;: &#123;&quot;$elemMatch&quot;: &#123;&quot;Uin&quot;: <span class="number">124</span>&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="slice">$slice</h4>
-<p>切片，只打印字段中数组前n个数据</p>
-<p>筛选出记录之后，只打印记录<code>score</code>字段的前1条数据</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123;&quot;_id&quot;: <span class="number">1</span>, &quot;score&quot;: &#123;&quot;$slice&quot;: <span class="number">1</span>&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<p>输出如下</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h4 id="regex">$regex</h4>
-<p>使用正则表达式，更多操作请查看官方文档: <a
-href="https://docs.mongodb.com/manual/reference/operator/query/regex/index.html">$regex</a></p>
-<p>找出<code>_id</code>字段中所有以a开头的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$regex&quot;: &quot;^a&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h3 id="后缀">后缀</h3>
-<h4 id="sort排序">sort——排序</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">1</span>&#125;) # 升序</span><br><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">-1</span>&#125;) # 降序</span><br></pre></td></tr></table></figure>
-<h4 id="limit限制">limit——限制</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().limit(<span class="number">3</span>) # 对查询结果只输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
-<h4 id="skip跳过">skip——跳过</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">skip</span>(<span class="number">3</span>) # 对查询结果跳过输出前<span class="number">3</span>个，即不输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
-<h4 id="count计数">count——计数</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="built_in">count</span>()  # 返回查询结果总数</span><br></pre></td></tr></table></figure>
-<h4
-id="collation基于特定语言进行操作">collation——基于特定语言进行操作</h4>
-<p>没有怎么看懂，我理解的大概意思就是说，运行根据特定语言来进行查找。感觉用的不多，这里贴上官方给的例子</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">collation</span>( &#123; locale: &quot;en_US&quot;, strength: <span class="number">1</span> &#125; )</span><br></pre></td></tr></table></figure>
-<h3 id="个人常用操作">个人常用操作</h3>
-<p>例子： <figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">// test_db</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> ObjectId(<span class="string">&quot;56063f17ade2f21f36b03133&quot;</span>)<span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;title&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 是一个 Nosql 数据库&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;by&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;菜鸟教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;url&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;http://www.runoob.com&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;tags&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">           <span class="string">&quot;mongodb&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;database&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;NoSQL&quot;</span></span><br><span class="line">   <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@a1bf7a55ad8478017b95124774687769&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">12266535</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;一二三&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@5571e120b5d8f54f93338c76f62b9c419b818c29373dca9f331c2911a203bfbe&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> NumberLong(<span class="number">3379331828</span>)<span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;灵灵八&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure></p>
-<h4 id="返回查找结果的数目">返回查找结果的数目</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;title&quot;: &quot;MongoDB 教程&quot;&#125;).<span class="built_in">count</span>()</span><br></pre></td></tr></table></figure>
-<h4
-id="查找某个key中的数组是否某包含某内容">查找某个key中的数组是否某包含某内容</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;tags&quot;: &#123;&quot;$in&quot;: [&quot;mongodb&quot;]&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="使用正则进行查找">使用正则进行查找</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;url&quot;: &#123;&quot;$regex&quot;: &quot;^http&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="查找某个由数组构成的key方法">查找某个由数组构成的key方法</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;links&quot;:&#123;&quot;$elemMatch&quot;:&#123;&quot;Uin&quot;:<span class="number">12266535</span>,&quot;NickName&quot;:&quot;一二三&quot;&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h2 id="附录">附录</h2>
-<h3 id="数据库权限说明表">数据库权限说明表</h3>
-<table>
-<colgroup>
-<col style="width: 23%" />
-<col style="width: 76%" />
-</colgroup>
-<thead>
-<tr>
-<th>名称</th>
-<th style="text-align: left;">权限</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>数据库用户角色</td>
-<td style="text-align: left;">read、readWrite</td>
-</tr>
-<tr>
-<td>数据库管理角色</td>
-<td style="text-align: left;">dbAdmin、dbOwner、userAdmin</td>
-</tr>
-<tr>
-<td>集群管理角色</td>
-<td
-style="text-align: left;">clusterAdmin、clusterManager、clusterMonitor、hostManager</td>
-</tr>
-<tr>
-<td>备份恢复角色</td>
-<td style="text-align: left;">backup、restore</td>
-</tr>
-<tr>
-<td>所有数据库角色</td>
-<td
-style="text-align: left;">readAnyDatabase、readWriteAnyDatabase、userAdminAnyDatabase、dbAdminAnyDatabase</td>
-</tr>
-<tr>
-<td>超级用户角色</td>
-<td style="text-align: left;">root</td>
-</tr>
-<tr>
-<td>提供了系统超级用户的访问</td>
-<td style="text-align: left;">dbOwner
-、userAdmin、userAdminAnyDatabase</td>
-</tr>
-<tr>
-<td>内部角色</td>
-<td style="text-align: left;">__system</td>
-</tr>
-</tbody>
-</table>
-<p>下一篇：<a href="/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/" title="Mongodb：副本集配置">Mongodb：副本集配置</a></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E7%9A%84%E9%80%89%E6%8B%A9.png" />
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E6%9C%80%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E5%81%9C%E6%AD%A2%E5%87%86%E5%88%99.png" /></p>
+<h3 id="例子">例子</h3>
+<p>非凸函数</p>
+<p><span class="math inline">\(\int _ { 0 } ^ { x 2 } ( \frac { e ^ { x
+_ { 3 } ^ { 2 } z } \cos ( z ) } { x _ { 3 } ^ { 4 } z + 1 } + z ^ { x 1
+} ) d z\)</span></p>
+<p><span class="math inline">\(e ^ { x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2
+} } + x _ { 1 } ^ { 2 } x _ { 2 } ^ { 2 }\)</span></p>
+<p><span class="math inline">\(( x _ { 1 } + x _ { 2 } ) ^ { 3 } - \log
+( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )\)</span></p>
+<p>$x _ { 1 } ^ { 4 } + x _ { 2 } ^ { 2 } + 3 x _ { 1 } x _ { 3 } + 2 x
+_ { 4 } x _ { 5 } $</p>
+<p>$9 x _ { 1 } ^ { 2 } + 4 x _ { 2 } ^ { 2 } + 2 x _ { 1 } x _ { 2 } +
+8 x _ { 3 } ^ { 2 } - 2 x _ { 1 } x _ { 3 } $</p>
+<p>$ { ( x _ { 1 } + 3 x _ { 2 } + 4 x _ { 3 } ) ^ { 2 } } + ( ( x _ { 1
+} + 4 x _ { 2 } + 5 x _ { 3 } - 100 ) ^ { 4 } )$</p>
+<p>$ { 1 + } $</p>
+<p>凸函数</p>
+<p>$( x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2 } ) + 14 x _ { 2 } ^ { 4 }
+$</p>
+<p>$ ( ( x _ { 1 } - 1 ) ^ { 2 } + ( x _ { 2 } - 2 ) ^ { 2 } ) ^ { 2 } -
+( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )$</p>
+<p>$| 2 x _ { 1 } + 4 x _ { 2 } + 3 x _ { 3 } + 2 x _ { 4 } + 7 | + ( (
+x _ { 1 } + x _ { 2 } - 4 x _ { 3 } - x _ { 4 } ) ^ { 4 } ) $</p>
+<p>$( F _ { 1 } ( x ) , 2 F _ { 2 } ( x ) , F _ { 3 } ( x ) + 8 F _ { 4
+} ( x ) ) $</p>
+<p>$ x _ { 1 } + | x _ { 2 } | + 3 x _ { 3 } + 4 | x _ { 4 } |$</p>
+<p><span class="math inline">\(( 4 x _ { 1 } + 7 x _ { 2 } - 4 x _ { 3 }
++ 12 x _ { 4 } ) ^ { 3 }\)</span></p>
+<p><span class="math inline">\(( x _ { 1 } - 5 ) ^ { 2 } + ( x _ { 2 } -
+8 ) ^ { 4 }\)</span></p>
+<p>线性规划问题</p>
+<p><span class="math inline">\(| x _ { 1 } - 4 x _ { 2 } + x _ { 3 } - 4
+|\)</span></p>
+<p>$( 4 x _ { 1 } - 3 x _ { 2 } + 8 x _ { 3 } - 5 , - 2 x _ { 1 } + 7 x
+_ { 2 } - x _ { 3 } + 1 ) $</p>
+<p>二次规划（凸二次函数）</p>
+<p><span class="math inline">\(2 ^ { 5 x _ { 1 } ^ { 2 } + 2 x _ { 2 } ^
+{ 2 } + 6 x _ { 1 } x _ { 2 } + 3 x _ { 1 } - x _ { 2 } }\)</span></p>
+<p><span class="math inline">\(1 - 4 x _ { 1 } ^ { 2 } - 9 x _ { 2 } ^ {
+2 } - 8 x _ { 3 } ^ { 2 } + 4 x _ { 1 } x _ { 2 } - 2 x _ { 1 } x _ { 3
+} + 4 x _ { 2 } x _ { 3 }\)</span></p>
+<h3 id="相关书籍">相关书籍</h3>
+<p>[1] 程理民, 吴江, 张玉林. 运筹学模型与方法教程[M]. 清华大学出版社,
+2000.</p>
+<p>[2] 陈宝林. 最优化理论与算法[M]. 清华大学出版社有限公司, 2005.</p>
+<p>[3] 袁亚湘, 孙文瑜. 最优化理论与方法[M]. 科学出版社, 1997.</p>
+<p>个人感觉，难度[3]&gt;[2]&gt;[1]，袁院士那本理论证明偏多，可搭配[2]与课后练习一起阅读。</p>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>mongodb</tag>
-        <tag>数据库</tag>
+        <tag>Optimization</tag>
       </tags>
   </entry>
   <entry>
-    <title>SQL语句的优化</title>
-    <url>/hexo_blog/2018/04/07/%E6%95%B0%E6%8D%AE%E5%BA%93/SQL%E8%AF%AD%E5%8F%A5%E7%9A%84%E4%BC%98%E5%8C%96/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了关于SQL语句（主要是Oracle数据库）的优化方案。
-<span id="more"></span></p>
-<p><a
-href="https://juejin.im/post/5ab0f39df265da23866fba55">数据库性能优化之SQL语句优化</a></p>
-<h2 id="操作符优化">操作符优化</h2>
-<h3 id="in-操作符">IN 操作符</h3>
-<p>优点：编写容易，清晰易懂</p>
-<p>缺点：性能较低。</p>
-<p>Oracle执行步骤：</p>
-<ol type="1">
-<li>将其转换多表连接</li>
-<li>如果转换不成功则先执行IN里面的子查询，再执行外层表记录</li>
-<li>如果转换成功则直接采用多表连接查询</li>
-</ol>
-<p>所以用IN的SQL至少会多一个转换过程。</p>
-<p>替代方案：在业务密集的SQL用EXISTS替代</p>
-<h3 id="not-in操作符">NOT IN操作符</h3>
-<p>不能应用表的索引，不建议使用。</p>
-<p>替代方案：用NOT EXISTS方案</p>
-<h3 id="is-null或is-not-null操作">IS NULL或IS NOT NULL操作</h3>
-<p>判断字段为空一般是不会用索引的（索引不索引空值）。如果某列存在空值，即使对该列建索引也不会提高性能。</p>
-<p>替代方案：将<code>a is not null</code>改为<code>a&gt;0</code>或<code>a&gt;' '</code>等。不允许字段为空，而用一个缺省值代替空值。</p>
-<h3 id="及-操作符">&lt; 及 &gt; 操作符</h3>
-<p>一般情况下是不需要调整的，在某些情况下可以进行优化。</p>
-<p>如：一个表有100万记录，一个数值型字段A，有30万的A=0，39万A=2，
-1万A=3。在这种情况下执行A&gt;2与A&gt;=3有很大区别，<code>A&gt;2</code>在Oracle中会先找出为2的记录索引再进行比较，但是<code>A&gt;=3</code>会直接找到等于3的记录索引。</p>
-<h3 id="like操作符">LIKE操作符</h3>
-<p>LIKE医用通配符查询，基本上可以达到任一查询。但是不同语句可能会产生性能上的很大差异。如<code>LIKE '%5400%'</code>这种情况下不会引用索引，<code>LIKE 'X5400%'</code>会引用范围索引。即前者会产生全表扫描，后者会进行范围内的查询，后者相对于前者的性能大大提高了。</p>
-<h3 id="union操作符">UNION操作符</h3>
-<p>UNION在进行表连接之后会筛选掉重复的记录，所以在表连接之后会对产生的结果集进行排序运算，删除重复记录再返回结果。但是实际上大部分应用中是不会产生重复的记录，最常见的是过程表于历史表UNION。在这种情况下，替代方案就是使用UNION
-ALL代替UNION。UNION ALL操作只是简单的将两个结果合并后就返回</p>
-<h3 id="联接列">联接列</h3>
-<p>对于有联接的列，优化器是不会使用索引的。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 使用联接查询</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employss <span class="keyword">where</span> first_name<span class="operator">||</span><span class="string">&#x27;&#x27;</span><span class="operator">||</span>last_name <span class="operator">=</span><span class="string">&#x27;Beill Cliton&#x27;</span>;</span><br><span class="line"># 不使用联接查询，基于last_name创建索引</span><br><span class="line"><span class="keyword">where</span> first_name <span class="operator">=</span><span class="string">&#x27;Beill&#x27;</span> <span class="keyword">and</span> last_name <span class="operator">=</span><span class="string">&#x27;Cliton&#x27;</span>;</span><br></pre></td></tr></table></figure>
-<h3 id="order-by语句">Order By语句</h3>
-<p>将返回的查询结果进行排序。任何在Order
-by语句的非索引项或者有计算表达式都将降低查询速度。解决方案就是为所使用的列建立另一个索引，同时也绝对避免在order
-by子句中使用表达式。</p>
-<h3 id="not-语句">NOT 语句</h3>
-<p>NOT可用来对任何逻辑运算符号取反。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 如果要使用<span class="keyword">NOT</span>，则应在取反的短语前面机上括号，并在短语前面机上<span class="keyword">NOT</span>运算符。</span><br><span class="line"><span class="keyword">where</span> <span class="keyword">not</span> (status <span class="operator">=</span> <span class="string">&#x27;VALID&#x27;</span>)</span><br><span class="line"># <span class="keyword">NOT</span>运算符包含在另外一个逻辑运算符<span class="operator">&lt;&gt;</span>运算符中。即使不在查询的<span class="keyword">where</span>子句中显式地加入<span class="keyword">NOT</span>词，<span class="keyword">NOT</span>仍在运算符中</span><br><span class="line"><span class="keyword">where</span> status <span class="operator">&lt;&gt;</span> <span class="string">&#x27;INVALID&#x27;</span>;</span><br><span class="line"># 这种情况运行建立salary的索引，在速度上会优于上面的方案</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employee <span class="keyword">where</span> salary <span class="operator">&lt;</span> <span class="number">3000</span> <span class="keyword">or</span> salary <span class="operator">&gt;</span> <span class="number">3000</span>;</span><br></pre></td></tr></table></figure>
-<h2 id="书写的影响">书写的影响</h2>
-<h3
-id="同一功能同一性能不同写法sql的影响">同一功能同一性能不同写法SQL的影响</h3>
-<p>尽可能避免不同写法的出现，根据Oracle共享内存SGA原理，如果语句不同则进行分析，占用共享内存。如果SQL字符串及格式完全相同，则Oracle只会分析一次，共享内存也只会留下一次的分析结果</p>
-<h3 id="where后面的条件顺序影响">WHERE后面的条件顺序影响</h3>
-<p>下面两个语句，在没有建立索引的情况下，执行都是全表扫描。</p>
-<p><code>dy_dj = '1KV以下'</code>这个条件在记录集内比率为99%,
-<code>xh_bz=1</code>而这个条件只有0.5%。所以第二条SQL的CPU占用率明显低于第一条。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span> <span class="keyword">and</span> xh_bz<span class="operator">=</span><span class="number">1</span></span><br><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> xh_bz<span class="operator">=</span><span class="number">1</span> <span class="keyword">and</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="查询表顺序的影响">查询表顺序的影响</h3>
-<p>在FROM后面的表中的列表顺序会对SQL执行性能影响，在没有索引及ORACLE没有对表进行统计分析的情况下，ORACLE会按<strong>表出现的顺序</strong>进行连接，所以如果表的顺序不对时会产生十分消耗服务器资源的数据交叉（如果对表进行了统计分析，Oracle自动先进小表的连接，再进行大表的连接）</p>
-<h2 id="sql语句索引的利用">SQL语句索引的利用</h2>
-<h3 id="条件字段的优化">条件字段的优化</h3>
-<p>函数处理的字段不能利用索引</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：substr(hbs_bh,<span class="number">1</span>,<span class="number">4</span>)<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span></span><br><span class="line">优化处理：hbs_bh <span class="keyword">like</span> <span class="string">&#x27;5400%&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：trunc(sk_rq)<span class="operator">=</span>trunc(sysdate)</span><br><span class="line">优化处理：sk_rq<span class="operator">&gt;=</span>trunc(sysdate) <span class="keyword">and</span> sk_rq<span class="operator">&lt;</span>trunc(sysdate<span class="operator">+</span><span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>进行了显示或隐式的运算的字段不能进行索引。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：<span class="string">&#x27;X&#x27;</span> <span class="operator">||</span> hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;X5400021452&#x27;</span></span><br><span class="line">优化处理：hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;5400021542&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：sk_rq<span class="operator">+</span><span class="number">5</span><span class="operator">=</span>sysdate</span><br><span class="line">优化处理：sk_rq<span class="operator">=</span>sysdate<span class="number">-5</span></span><br></pre></td></tr></table></figure>
-<p>条件内包括的多个本表的字段运算时不能进行索引</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：ys_df<span class="operator">&gt;</span>cx_df</span><br><span class="line">无法进行优化</span><br><span class="line"></span><br><span class="line">原语句：qc_bh <span class="operator">||</span> kh_bh<span class="operator">=</span><span class="string">&#x27;5400250000&#x27;</span></span><br><span class="line">优化处理：qc_bh<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span> <span class="keyword">and</span> kh_bh<span class="operator">=</span><span class="string">&#x27;250000&#x27;</span></span><br></pre></td></tr></table></figure>
-<h2 id="其他优化方法">其他优化方法</h2>
-<ol type="1">
-<li><p>选择最有效率的表名顺序（只在基于规则的优化器中有效）</p>
-<p><code>ORACLE</code>
-的解析器按照从右到左的顺序处理FROM子句中的表名，FROM子句中写在最后的表(基础表
-driving
-table)将被最先处理，在FROM子句中包含多个表的情况下,你必须选择记录条数最少的表作为基础表。如果有3个以上的表连接查询,
-那就需要选择交叉表(intersection table)作为基础表,
-交叉表是指那个被其他表所引用的表</p></li>
-<li><p><code>WHERE</code>子句中的连接顺序</p>
-<p><code>ORACLE</code>采用自下而上的顺序解析WHERE子句,根据这个原理,表之间的连接必须写在其他<code>WHERE</code>条件之前,
-那些可以过滤掉最大数量记录的条件必须写在WHERE子句的末尾.</p></li>
-<li><p><code>SELECT</code>子句中避免使用 '*'</p>
-<p><code>ORACLE</code>在解析的过程中, 会将'*'依次转换成所有的列名,
-这个工作是通过查询数据字典完成的, 这意味着将耗费更多的时间。</p></li>
-<li><p>减少访问数据库的次数</p>
-<p><code>ORACLE</code>在内部执行了许多工作: 解析SQL语句,
-估算索引的利用率, 绑定变量 , 读数据块等。</p></li>
-<li><p>在<code>SQL*Plus</code> ,
-<code>SQL*Forms</code>和<code>Pro*C</code>中重新设置<code>ARRAYSIZE</code>参数,
-可以增加每次数据库访问的检索数据量 ,建议值为200</p></li>
-<li><p>使用<code>DECODE</code>函数来减少处理时间</p>
-<p>使用<code>DECODE</code>函数可以避免重复扫描相同记录或重复连接相同的表.</p></li>
-<li><p>整合简单,无关联的数据库访问</p></li>
-</ol>
-<p>如果你有几个简单的数据库查询语句,你可以把它们整合到一个查询中(即使它们之间没有关系)
-。</p>
-<ol start="8" type="1">
-<li>删除重复记录：</li>
-</ol>
-<blockquote>
-<p>最高效的删除重复记录方法(因为使用了<code>ROWID</code>)</p>
-</blockquote>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">DELETE</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.ROWID <span class="operator">&gt;</span> (<span class="keyword">SELECT</span> <span class="built_in">MIN</span>(X.ROWID) <span class="keyword">FROM</span> EMP X <span class="keyword">WHERE</span> X.EMP_NO <span class="operator">=</span> E.EMP_NO)</span><br></pre></td></tr></table></figure>
-<ol start="9" type="1">
-<li><p>用<code>TRUNCATE</code>替代<code>DELETE</code></p>
-<p>当删除表中的记录时,在通常情况下, 回滚段(rollback segments )
-用来存放可以被恢复的信息.
-如果你没有<code>COMMIT</code>事务,<code>ORACLE</code>会将数据恢复到删除之前的状态(准确地说是恢复到执行删除命令之前的状况)
-而当运用<code>TRUNCATE</code>时,
-回滚段不再存放任何可被恢复的信息.当命令运行后,数据不能被恢复.因此很少的资源被调用,执行时间也会很短.
-(译者按:
-<code>TRUNCATE</code>只在删除全表适用,<code>TRUNCATE</code>是DDL不是DML)
-。</p></li>
-<li><p>尽量多使用<code>COMMIT</code></p>
-<p>只要有可能,在程序中尽量多使用<code>COMMIT</code>,
-这样程序的性能得到提高,需求也会因为<code>COMMIT</code>所释放的资源而减少，<code>COMMIT</code>所释放的资源:</p>
-<ol type="a">
-<li><p>回滚段上用于恢复数据的信息.</p></li>
-<li><p>被程序语句获得的锁</p></li>
-<li><p>redo log buffer 中的空间</p></li>
-<li><p><code>ORACLE</code>为管理上述3种资源中的内部花费</p></li>
-</ol></li>
-<li><p>用<code>Where</code>子句替换<code>HAVING</code>子句</p></li>
-</ol>
-<pre><code>避免使用`HAVING`子句, `HAVING` 只会在检索出所有记录之后才对结果集进行过滤. 这个处理需要排序,总计等操作. 如果能通过`WHERE`子句限制记录的数目,那就能减少这方面的开销. (非`oracle`中)`on`、`where`、`having`这三个都可以加条件的子句中，`on`是最先执行，`where`次之，`having`最后，因为`on`是先把不符合条件的记录过滤后才进行统计，它就可以减少中间运算要处理的数据，按理说应该速度是最快的，`where`也应该比`having`快点的，因为它过滤数据后才进行`sum`，在两个表联接时才用`on`的，所以在一个表的时候，就剩下`where`跟`having`比较了。在这单表查询统计的情况下，如果要过滤的条件没有涉及到要计算字段，那它们的结果是一样的，只是`where`可以使用`rushmore`技术，而`having`就不能，在速度上后者要慢如果要涉及到计算的字 段，就表示在没计算之前，这个字段的值是不确定的，根据上篇写的工作流程，`where`的作用时间是在计算之前就完成的，而`having`就是在计算后才起作 用的，所以在这种情况下，两者的结果会不同。在多表联接查询时，`on`比`where`更早起作用。系统首先根据各个表之间的联接条件，把多个表合成一个临时表 后，再由`where`进行过滤，然后再计算，计算完后再由`having`进行过滤。由此可见，要想过滤条件起到正确的作用，首先要明白这个条件应该在什么时候起作用，然后再决定放在那里。</code></pre>
-<ol start="12" type="1">
-<li>减少对表的查询</li>
-</ol>
-<blockquote>
-<p>在含有子查询的SQL语句中,要特别注意减少对表的查询</p>
-</blockquote>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> TAB_NAME <span class="keyword">FROM</span> TABLES <span class="keyword">WHERE</span> (TAB_NAME,DB_VER) <span class="operator">=</span> ( <span class="keyword">SELECT</span> TAB_NAME,DB_VER <span class="keyword">FROM</span> TAB_COLUMNS <span class="keyword">WHERE</span> VERSION <span class="operator">=</span> <span class="number">604</span>)</span><br></pre></td></tr></table></figure>
-<ol start="13" type="1">
-<li><p>通过内部函数提高SQL效率</p>
-<p>复杂的SQL往往牺牲了执行效率.
-能够掌握上面的运用函数解决问题的方法在实际工作中是非常有意义的。</p></li>
-<li><p>使用表的别名(<code>Alias</code>)：</p></li>
-</ol>
-<pre><code>当在SQL语句中连接多个表时, 请使用表的别名并把别名前缀于每个Column上.这样一来,就可以减少解析的时间并减少那些由Column歧义引起的语法错误。</code></pre>
-<ol start="15" type="1">
-<li>识别’低效执行’的SQL语句</li>
-</ol>
-<pre><code>虽然目前各种关于SQL优化的图形化工具层出不穷,但是写出自己的SQL工具来解决问题始终是一个最好的方法。</code></pre>
-<p>​<code>sql SELECT EXECUTIONS, DISK_READS, BUFFER_GETS, ROUND((BUFFER_GETS-DISK_READS)/BUFFER_GETS,2) Hit_radio, ROUND(DISK_READS/EXECUTIONS,2) Reads_per_run, SQL_TEXT FROM V$SQLAREA WHERE EXECUTIONS&gt;0 AND BUFFER_GETS &gt; 0 AND (BUFFER_GETS-DISK_READS)/BUFFER_GETS &lt; 0.8 ORDER BY 4 DESC; ​</code></p>
-<ol start="16" type="1">
-<li><p>用索引提高效率</p>
-<p>索引是表的一个概念部分,用来提高检索数据的效率，ORACLE使用了一个复杂的自平衡B-tree结构.
-通常,通过索引查询数据比全表扫描要快.
-当ORACLE找出执行查询和Update语句的最佳路径时, ORACLE优化器将使用索引.
-同样在联结多个表时使用索引也可以提高效率.
-另一个使用索引的好处是,它提供了主键(primary
-key)的唯一性验证.。那些LONG或LONG RAW数据类型, 你可以索引几乎所有的列.
-通常, 在大型表中使用索引特别有效. 当然,你也会发现,
-在扫描小表时,使用索引同样能提高效率.
-虽然使用索引能得到查询效率的提高,但是我们也必须注意到它的代价.
-索引需要空间来存储,也需要定期维护, 每当有记录在表中增减或索引列被修改时,
-索引本身也会被修改. 这意味着每条记录的INSERT , DELETE ,
-UPDATE将为此多付出4 , 5 次的磁盘I/O .
-因为索引需要额外的存储空间和处理,那些不必要的索引反而会使查询反应时间变慢.。定期的重构索引是有必要的</p></li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">ALTER</span> INDEX <span class="operator">&lt;</span>INDEXNAME<span class="operator">&gt;</span> REBUILD <span class="operator">&lt;</span>TABLESPACENAME<span class="operator">&gt;</span></span><br></pre></td></tr></table></figure>
-<ol start="17" type="1">
-<li><p>用<code>EXISTS</code>替换<code>DISTINCT</code></p>
-<p>当提交一个包含一对多表信息(比如部门表和雇员表)的查询时,避免在<code>SELECT</code>子句中使用<code>DISTINCT</code>.
-一般可以考虑用<code>EXIST</code>替换, <code>EXISTS</code>
-使查询更为迅速,因为RDBMS核心模块将在子查询的条件一旦满足后,立刻返回结果.
-例子：</p></li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">(低效):</span><br><span class="line"><span class="keyword">SELECT</span> <span class="keyword">DISTINCT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D, EMP E <span class="keyword">WHERE</span> D.DEPT_NO <span class="operator">=</span> E.DEPT_NO</span><br><span class="line">(高效):</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D <span class="keyword">WHERE</span> <span class="keyword">EXISTS</span> ( <span class="keyword">SELECT</span> <span class="string">&#x27;X&#x27;</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.DEPT_NO <span class="operator">=</span> D.DEPT_NO);</span><br></pre></td></tr></table></figure>
-<ol start="18" type="1">
-<li><p>sql语句用大写的</p>
-<p>因为oracle总是先解析sql语句，把小写的字母转换成大写的再执行</p></li>
-<li><p>在java代码中尽量少用连接符“＋”连接字符串！</p></li>
-<li><p>避免在索引列上使用NOT，通常我们要避免在索引列上使用NOT,
-NOT会产生在和在索引列上使用函数相同的影响.
-当ORACLE”遇到”NOT,他就会停止使用索引转而执行全表扫描。</p></li>
-<li><p>避免在索引列上使用计算</p>
-<p>WHERE子句中，如果索引列是函数的一部分．优化器将不使用索引而使用全表扫描．</p></li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">*</span> <span class="number">12</span> <span class="operator">&gt;</span> <span class="number">25000</span>;</span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">&gt;</span> <span class="number">25000</span><span class="operator">/</span><span class="number">12</span>;</span><br></pre></td></tr></table></figure>
-<ol start="22" type="1">
-<li>用&gt;=替代&gt;</li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;=</span> <span class="number">4</span></span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;</span> <span class="number">3</span></span><br></pre></td></tr></table></figure>
-<ol start="23" type="1">
-<li><p>用UNION替换OR (适用于索引列)</p>
-<p>通常情况下, 用UNION替换WHERE子句中的OR将会起到较好的效果.
-对索引列使用OR将造成全表扫描. 注意, 以上规则只针对多个索引列有效.
-如果有column没有被索引, 查询效率可能会因为你没有选择OR而降低.
-在下面的例子中, LOC_ID 和REGION上都建有索引.</p></li>
+    <title>拉格朗日乘子法：从函数的求导到不等式约束的求解</title>
+    <url>/hexo_blog/2018/04/30/%E6%95%B0%E5%AD%A6/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>本篇文章简单梳理了关于拉格朗日乘子法的相关问题及一些相关概念的介绍。</p>
+<span id="more"></span>
+<h3 id="基本概念">基本概念</h3>
+<p>拉格朗日乘子法（Lagrange
+Multiplier）在某些地方又叫做拉格朗日乘数法。在求解最优化问题中，这个方法是最常用的方法之一。最优化问题往往抽象为某个函数，即求函数的最大值或最小值。求解最大值和最小值的求解可以相互转换（对偶）。对于求解最优化问题分为以下三种情况（难度依此递增）：</p>
+<h3 id="无约束条件">无约束条件</h3>
+<p>一般情况下，只需要对给出的函数进行求导，令其导函数为0的点可能就是极值点，将可能的结果分别代回原函数进行验证即可。</p>
+<p>假设有这样一个函数<span class="math inline">\(f(x)\)</span>，令<span
+class="math inline">\(g(x)=f^{&#39;}(x)\)</span>，<span
+class="math inline">\(g(x)=0\)</span>的解有<span
+class="math inline">\(x_1,x_2,x_3,\cdots\)</span>。求函数<span
+class="math inline">\(f(x)\)</span>的最优解就只需要分别将<span
+class="math inline">\(x_1,x_2,x_3,\cdots\)</span>代入<span
+class="math inline">\(f(x)\)</span>，比较大小，得到最优解<span
+class="math inline">\(x_i\)</span>。</p>
+<p>从这里开始引出<strong>拉格朗日乘子法</strong></p>
+<h3 id="等式约束条件">等式约束条件</h3>
+<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
+约束条件为$h_k(x) $, <span
+class="math inline">\(k=1,2,\cdots,l\)</span>，即要求<span
+class="math inline">\(f(x)\)</span>的最优解，但是有<span
+class="math inline">\(l\)</span>个<span
+class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
+<p><span class="math display">\[
+F(x, \lambda)=f(x)+\sum\limits_{k=1}^{l}\lambda_kh_k(x)
+\]</span></p>
+<p><span
+class="math inline">\(\lambda_k\)</span>表示约束条件的待定系数。之后解变量的偏导方程，如下:</p>
+<p><span class="math display">\[
+\frac{\partial F}{\partial x_k}=0, \frac{\partial F}{\partial
+\lambda_k}=0, k=1,2,\cdots, l
+\]</span></p>
+<p>求解得到了<span
+class="math inline">\(l+1\)</span>个方程，将结果代回原方程验证即可得到解。</p>
+<p>举例：</p>
+<p>给定一个椭圆<span
+class="math inline">\(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}=1\)</span>，求解<span
+class="math inline">\(f(x,y,z)=8xyz\)</span>的最大值。</p>
+<p>求解过程如下：</p>
+<ul>
+<li><p>通过拉格朗日乘子法的定义，得到如下方程</p>
+<p><span class="math display">\[
+\begin{array}{l}F(x,y,z,\lambda)&amp;=f(x,y,z)+\lambda\cdot h(x,y,z) \\
+&amp;=8xyz+\lambda\cdot(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1)
+\end{array}
+\]</span></p></li>
+<li><p>接下来对<span
+class="math inline">\(F(x,y,z,\lambda)\)</span>求偏导，分别得到</p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial x}=8yz+\frac{2\lambda
+x}{a^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial y}=8xz+\frac{2\lambda
+y}{b^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial z}=8xy+\frac{2\lambda
+z}{c^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial
+\lambda}=\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1=0
+\]</span></p></li>
+<li><p>联立前三个方程可以得到<span
+class="math inline">\(bx=ay\)</span>和<span
+class="math inline">\(az=cx\)</span>，代入最后一个方程，解得：</p>
+<p><span class="math display">\[
+x=\frac{\sqrt{3}}{3}a
+\]</span></p>
+<p><span class="math display">\[
+y=\frac{\sqrt{3}}{3}b
+\]</span></p>
+<p><span class="math display">\[
+z=\frac{\sqrt{3}}{3}c
+\]</span></p></li>
+<li><p>最后代入原方程，解得最大体积</p>
+<p><span class="math display">\[
+V_{max}=f(\frac{\sqrt{3}}{3}a,\frac{\sqrt{3}}{3}b,\frac{\sqrt{3}}{3}c)=\frac{8\sqrt{3}}{9}abc
+\]</span></p></li>
+</ul>
+<h3 id="不等式约束条件">不等式约束条件</h3>
+<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
+不等式约束条件为<span class="math inline">\(c_i(x)\)</span>,<span
+class="math inline">\(i=1,2,\cdots,k\)</span>，等式约束条件为$h_j(x) $,
+<span class="math inline">\(j=1,2,\cdots,l\)</span>，即要求<span
+class="math inline">\(f(x)\)</span>的最优解，但是有<span
+class="math inline">\(k\)</span>个<span
+class="math inline">\(g(x)\)</span>不等式和<span
+class="math inline">\(l\)</span>个<span
+class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
+<p><span class="math display">\[
+L(x,a,b)=f(x)+\sum\limits^{k}_{i=1}a_ic_i(x)+\sum\limits^{l}_{j=1}b_jh_j(x)
+\]</span></p>
+<p>这里用一个叫做KKT条件的方法，把所有的不等式约束、等式约束和目标函数全部写为一个式子<span
+class="math inline">\(L(a,b,x)=f(x)+a*c(x)+b*h(x)\)</span>。</p>
+<p>KKT条件是指最优解必须满足以下条件：</p>
+<ul>
+<li><span class="math inline">\(L(a,b,x)\)</span>求<span
+class="math inline">\(x\)</span>求导为0</li>
+<li><span class="math inline">\(h(x)=0\)</span></li>
+<li><span class="math inline">\(a*c(x)=0\)</span></li>
+</ul>
+<p>这样就可以将不等式约束转换为了等式约束，求<span
+class="math inline">\(L(a,b,x)\)</span>的偏导等于零即可求得最优参数。</p>
+<p><span class="math display">\[
+\min f(x)\Leftrightarrow \min\limits_x \max\limits_{a,b} L(x,a,b)
+\]</span></p>
+<p>经过对偶变换得：</p>
+<p><span class="math display">\[
+\max\limits_{a,b} \min L(x,a,b)
+\]</span></p>
+<p>因为<span class="math inline">\(c(x)&lt;0\)</span>，所以只有当<span
+class="math inline">\(a*c(x)=0\)</span>时，<span
+class="math inline">\(L(x,a,b)\)</span>才能取得最大值。以上就可以使用同等式约束的求解方法进行求解。</p>
+<h3 id="补充部分">补充部分</h3>
+<h4 id="定义不等式约束问题的标准写法">定义不等式约束问题的标准写法</h4>
+<p><span class="math inline">\(f(x), c_i(x),h_j(x)\)</span>是定义在<span
+class="math inline">\(R^n\)</span>上的连续可微函数，考虑约束最优化问题：</p>
+<p><span class="math inline">\(\min f(x)  \quad  x\in R^n\)</span></p>
+<p><span class="math inline">\(s.t. c_i(x) \leq 0 \quad i =
+1...k\)</span></p>
+<pre><code>$h_j(x)=0 \quad j = 1...l$</code></pre>
+<p>即有<span class="math inline">\(k\)</span>个不等式约束<span
+class="math inline">\(c_i(x)\)</span>和<span
+class="math inline">\(l\)</span>个等式约束<span
+class="math inline">\(h_j(x)\)</span></p>
+<h4 id="拉格朗日函数">拉格朗日函数</h4>
+<p><span class="math inline">\(L(x,\alpha,
+\beta)=f(x)+\sum^{k}_{i=1}\alpha_ic_i(x)+\sum^{l}_{j=1}\beta_jh_j(x)\)</span></p>
+<p>其中的<span class="math inline">\(\alpha_i\)</span>和<span
+class="math inline">\(\beta_j\)</span>，称为<strong>拉格朗日乘子</strong>，<span
+class="math inline">\(\alpha_i \geq 0\)</span></p>
+<h4 id="几何意义">几何意义</h4>
+<p>对于一个二维空间来说，求解最短路径的问题。将维度升高，变为对于多维空间来说，求解最短路径问题。</p>
+<p>比较直观的解释，爬山的目标是到山顶，根据梯度上升（下降）的方法来寻求优解，即爬到山顶。</p>
+<h4 id="对偶概念的简单介绍">对偶概念的简单介绍</h4>
+<p>一对一的一种方式，把一种概念、公理或数学结构转换为另一种概念、公理或数学结构。如果A的对偶是B，那么B的对偶是A。A的对偶有时候是A的本身。</p>
+<p>有界线性泛函数：复杂对象映射到<strong>简单对象</strong>上</p>
+<p>群表示：简单对象具有类似线性结构的<strong>简单结构</strong></p>
+<p>范畴论：映射箭头所构成的集合</p>
+<p>（像集，映射）——&gt;原像集</p>
+<p>对联可以称为一种对偶，上联与下联有对应的关系（相似和取逆）。对偶是有一定要有自然合理的对应关系（或者说能够吹出来的自然合理关系）。</p>
+<p>常见的自然、合理法则首推：相似（包括同构）和取逆（包括互反）</p>
+<p>常见对偶的例子：</p>
+<ol type="1">
+<li>奇数和偶数</li>
+<li>洛必达法则和Stolz 定理（相当于离散洛必达法则）</li>
+<li>Fourier 系数与 Fourier 变换（傅里叶）</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br></pre></td></tr></table></figure>
-<ol start="24" type="1">
-<li><p>用IN来替换OR</p>
-<p>这是一条简单易记的规则，但是实际的执行效果还须检验，在ORACLE8下，两者的执行路径似乎是相同的．</p></li>
+<p>对偶可以理解为，一个问题不好理解转换为另一个简单方式的问题来理解，二者在本质上是相同的。</p>
+<p>比如：某个函数的最大化不方便求解，那么就求这个函数的最小化。</p>
+<p>未完待续……</p>
+<ol type="1">
+<li>罚函数</li>
+<li>广义乘子法</li>
+<li>KKT的求导证明</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span>…. <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">20</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">30</span></span><br><span class="line">高效</span><br><span class="line"><span class="keyword">SELECT</span>… <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_IN <span class="keyword">IN</span> (<span class="number">10</span>,<span class="number">20</span>,<span class="number">30</span>);</span><br></pre></td></tr></table></figure>
-<ol start="25" type="1">
-<li><p>避免在索引列上使用IS NULL和IS NOT NULL</p>
-<p>避免在索引中使用任何可以为空的列，ORACLE将无法使用该索引．对于单列索引，如果列包含空值，索引中将不存在此记录.
-对于复合索引，如果每个列都为空，索引中同样不存在此记录.
-如果至少有一个列不为空，则记录存在于索引中．举例:
-如果唯一性索引建立在表的A列和B列上,
-并且表中存在一条记录的A,B值为(123,null) ,
-ORACLE将不接受下一条具有相同A,B值（123,null）的记录(插入).
-然而如果所有的索引列都为空，ORACLE将认为整个键值为空而空不等于空.
-因此你可以插入1000 条具有相同键值的记录,当然它们都是空!
-因为空值不存在于索引列中,所以WHERE子句中对索引列进行空值比较将使ORACLE停用该索引.</p></li>
+]]></content>
+      <categories>
+        <category>MachineLearning</category>
+      </categories>
+      <tags>
+        <tag>数学</tag>
+        <tag>Optimization</tag>
+        <tag>拉格朗日乘子法</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>矩阵分解介绍及Python实现（QR、SVD、LU）</title>
+    <url>/hexo_blog/2018/04/13/%E6%95%B0%E5%AD%A6/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3%E4%BB%8B%E7%BB%8D%E5%8F%8APython%E5%AE%9E%E7%8E%B0%EF%BC%88QR%E3%80%81SVD%E3%80%81LU%EF%BC%89/</url>
+    <content><![CDATA[<p>本篇文章简单介绍了三种矩阵分解方式并且附上部分分解方式的实现代码。</p>
+<span id="more"></span>
+<h1 id="矩阵分解">矩阵分解</h1>
+<p>Python实现了QR与LU分解，QR用了两种变换方式实现。<a
+href="https://github.com/wnma3mz/Matrix-factorization">Github地址</a></p>
+<h2 id="what">What？</h2>
+<p><a
+href="https://zh.wikipedia.org/wiki/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3">wiki链接</a></p>
+<p>将矩阵拆分成数个三角形矩阵。</p>
+<h2 id="why">Why？</h2>
+<ol type="1">
+<li>数值分析中，用于实现一些矩阵运算的快速算法</li>
+<li>推荐算法（SVD），信号处理（SVD）</li>
+<li>反应矩阵中的一些数值特性：如矩阵的秩、特征值(奇异值)</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引失效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="keyword">IS</span> <span class="keyword">NOT</span> <span class="keyword">NULL</span>;</span><br><span class="line">高效: (索引有效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="operator">&gt;=</span><span class="number">0</span>;</span><br></pre></td></tr></table></figure>
-<ol start="26" type="1">
-<li><p>总是使用索引的第一个列</p>
-<p>如果索引是建立在多个列上, 只有在它的第一个列(leading
-column)被where子句引用时,优化器才会选择使用该索引.
-这也是一条简单而重要的规则，当仅引用索引的第二个列时,优化器使用了全表扫描而忽略了索引。</p></li>
-<li><p>用UNION-ALL 替换UNION ( 如果有可能的话)</p>
-<p>当SQL
-语句需要UNION两个查询结果集合时,这两个结果集合会以UNION-ALL的方式被合并,
-然后在输出最终结果前进行排序. 如果用UNION ALL替代UNION,
-这样排序就不是必要了. 效率就会因此得到提高. 需要注意的是，UNION ALL
-将重复输出两个结果集合中相同记录. 因此各位还是要从业务需求分析使用UNION
-ALL的可行性. UNION
-将对结果集合排序,这个操作会使用到SORT_AREA_SIZE这块内存.
-对于这块内存的优化也是相当重要的.
-下面的SQL可以用来查询排序的消耗量</p></li>
+<p>每一种矩阵分解方式对应不同的意义</p>
+<h2 id="how">How？</h2>
+<h3 id="基本概念介绍">基本概念介绍</h3>
+<p>关于转置、逆矩阵就不多介绍了。</p>
+<ul>
+<li>满秩矩阵
+<ul>
+<li>矩阵的秩等于行数—&gt;行满秩</li>
+<li>矩阵的秩等于列数—&gt;列满秩</li>
+<li>以上两个都满足称为n阶方阵</li>
+</ul></li>
+<li>酉矩阵（正交矩阵）
+<ul>
+<li>方块矩阵，元素皆为实数，行与列皆正交的单位向量</li>
+<li>特性：矩阵的转置矩阵为逆矩阵</li>
+<li>定理（det表示行列式）：</li>
+</ul></li>
+</ul>
+<p><span class="math display">\[Q^T=Q^{-1}\Leftrightarrow Q^TQ=QQ^T =
+I\]</span></p>
+<p><span
+class="math display">\[1=\det(I)=\det(Q^TQ)=\det(Q^T)\det(Q)=(\det(Q))^2\Rightarrow|Q|
+\pm 1\]</span></p>
+<ul>
+<li><p>单位矩阵</p>
+<ul>
+<li>矩阵中的每个元素都为1</li>
+<li>如下</li>
+</ul>
+<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 1
+&amp; 1 \\ 1 &amp; 1 &amp; 1 \\ 1&amp; 1 &amp; 1 \end{array}
+\right|\]</span></p></li>
+<li><p>上三角矩阵</p>
+<ul>
+<li>非零元素在右上方的矩阵</li>
+<li>如下</li>
+</ul>
+<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 2
+&amp; 3 \\ 0 &amp; 1 &amp; 2\\ 0&amp; 0 &amp; 1 \end{array}
+\right|\]</span></p></li>
+<li><p>二范数</p>
+<ul>
+<li>假设有一个x的向量</li>
+</ul>
+<p><span class="math display">\[x= (1, 2, 3)^T\]</span></p>
+<ul>
+<li>那么它的二范数表示如下</li>
+</ul>
+<p><span class="math display">\[||x||_2 =\sqrt{1^2 + 2^2 +
+3^2}=\sqrt{14}\]</span></p></li>
+<li><p>共轭转置</p>
+<p><span class="math display">\[A^*=(\bar{A})^T=\bar{A^T}\]</span></p>
+<ul>
+<li>现有矩阵A</li>
+</ul>
+<p><span class="math display">\[A=\left[ \begin{array}{ccc} 3+i &amp; 5
+\\ 2-2i &amp; i\end{array} \right]\]</span></p>
+<ul>
+<li>则它的共轭矩阵如下</li>
+</ul>
+<p><span class="math display">\[A^*=\left[ \begin{array}{ccc} 3-i &amp;
+2+2i \\ 5 &amp; -i\end{array} \right]\]</span></p></li>
+<li><p>特征值与奇异值</p>
+<p>我的理解是：特征值是奇异值的一种特殊情况，特征值是在矩阵是方阵时候的奇异值。</p>
+<p>关于什么是特征值。</p>
+<p>一个矩阵可以理解为由n个向量组成的，而特征值就是反应向量长度（重要性）的标准，特征值越大，则对应向量就越重要。</p>
+<p>假设判断电影为动作片还是搞笑片，可以由以下几个特征来衡量，打斗场景、观众笑声次数、导演、演员……。将数据离散化成矩阵，然后提取特征值，特征值越大的特征则对于衡量电影是动作片还是搞笑片的比重越大。</p></li>
+<li><p>Householder矩阵</p>
+<ul>
+<li><p><a
+href="https://zh.wikipedia.org/wiki/豪斯霍尔德变换">wiki链接</a></p></li>
+<li><p>公式如下，H表示Householder矩阵，I表示单位矩阵，v表示单位向量，v*表示v的共轭转置。</p>
+<p><span class="math display">\[H=I-2vv^*\]</span></p></li>
+<li><p>性质：</p>
+<ol type="1">
+<li>H矩阵的转置等于它本身的逆</li>
+<li>H矩阵的平方等于单位矩阵</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span> <span class="keyword">ALL</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br></pre></td></tr></table></figure>
-<ol start="28" type="1">
-<li><p>用WHERE替代ORDER BY</p>
-<p>ORDER BY 子句只在两种严格的条件下使用索引.</p>
-<p>ORDER BY中所有的列必须包含在相同的索引中并保持在索引中的排列顺序.</p>
-<p>ORDER BY中所有的列必须定义为非空.</p>
-<p>WHERE子句使用的索引和ORDER BY子句中所使用的索引不能并列.</p></li>
+<p><span class="math display">\[H^{-1}=H^*\]</span></p>
+<p><span class="math display">\[H^2=I\]</span></p></li>
+</ul></li>
+</ul>
+<h3 id="svd分解">1. SVD分解</h3>
+<p><a
+href="https://blog.csdn.net/u010099080/article/details/68060274">数学解释参考链接</a></p>
+<p>奇异值分解。公式如下：</p>
+<p><span class="math display">\[M=U\Sigma V^*\]</span></p>
+<p>举例：</p>
+<p><span class="math display">\[\begin{array}{clll} M &amp;= U\Sigma V^*
+\\ &amp;= \begin{bmatrix} 1 &amp; 0 &amp; 0 &amp; 0 &amp; 2\\ 0 &amp; 0
+&amp; 3 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 4
+&amp; 0 &amp; 0 &amp; 0\end{bmatrix} \\  &amp;= \begin{bmatrix} 0 &amp;
+0 &amp; 1 &amp; 0\\ 0 &amp; 1 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp;
+1\\ 1 &amp; 0 &amp; 0 &amp; 0\end{bmatrix} \cdot \begin{bmatrix} 4 &amp;
+0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 3 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp;
+0 &amp; \sqrt{5} &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp;
+0\end{bmatrix} \cdot \begin{bmatrix} 0 &amp; 1 &amp; 0 &amp; 0 &amp; 0\\
+0 &amp; 0 &amp; 1 &amp; 0 &amp; 0\\ \sqrt{0.2} &amp; 0 &amp; 0 &amp; 0
+&amp; \sqrt{0.8}\\ 0 &amp; 0 &amp; 0 &amp; 1 &amp; 0\\ \sqrt{0.8} &amp;
+0 &amp; 0 &amp; 0 &amp; -\sqrt{0.2}\end{bmatrix}
+\end{array}\]</span></p>
+<p>性质:</p>
+<ol type="1">
+<li>矩阵<span
+class="math inline">\(\Sigma\)</span>的所有非对角元为0。矩阵<strong>U</strong>和<span
+class="math inline">\(V^*\)</span>都是酉矩阵，它们乘上各自的共轭转置都得到单位矩阵</li>
+<li>奇异值的分解并不是唯一的，这是因为<span
+class="math inline">\(\Sigma\)</span>矩阵中的对角元有元素为0</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引不被使用)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">ORDER</span> <span class="keyword">BY</span> DEPT_TYPE</span><br><span class="line">高效: (使用索引)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> DEPT_TYPE <span class="operator">&gt;</span> <span class="number">0</span></span><br></pre></td></tr></table></figure>
-<ol start="29" type="1">
-<li><p>避免改变索引列的类型</p>
-<p>当比较不同数据类型的数据时,
-ORACLE自动对列进行简单的类型转换.</p></li>
+<p>步骤：</p>
+<ol type="1">
+<li>计算<span class="math inline">\(M.T\cdot
+M\)</span>得到特征向量（左奇异向量），是U的列向量</li>
+<li>计算<span class="math inline">\(M\cdot
+M.T\)</span>得到特征向量（右奇异向量），是V的列向量</li>
+<li>计算<span class="math inline">\(M\cdot M.T\)</span>或者<span
+class="math inline">\(M.T\cdot
+M\)</span>得到特征值，所有非零特征值的平方根（二范数）</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 假设 EMPNO是一个数值类型的索引列.</span><br><span class="line"># 低效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="string">&#x27;123&#x27;</span></span><br><span class="line"># 高效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="number">123</span></span><br></pre></td></tr></table></figure>
-<ol start="30" type="1">
-<li><p>需要当心的WHERE子句</p>
-<p>某些SELECT 语句中的WHERE子句不使用索引. 这里有一些例子.</p>
+<p>得到特征向量之后，需要依照特征值从大到小进行排序。</p>
+<p>关于求特征值的方法，线代书上已做介绍，就不赘述。如果运用编程方法求矩阵的特征值，最常见的方法有<strong>雅可比算法、幂法。</strong>当然也可以用先进行QR分解求得特征值之后，再继续进行SVD。</p>
+<h3 id="lu分解">2. LU分解</h3>
+<p><a
+href="https://jiayi797.github.io/2017/03/29/%E6%95%B0%E5%AD%A6-%E7%9F%A9%E9%98%B5LU%E5%88%86%E8%A7%A3/">参考链接</a></p>
+<p>将一个矩阵分解为一个下三角矩阵和一个上三角矩阵的乘积。形式如：</p>
+<p><span class="math display">\[A=LU\]</span></p>
+<p>格式如下</p>
+<p><span class="math display">\[\left[ \begin{array} { l l l } { a _ {
+11 } } &amp; { a _ { 12 } } &amp; { a _ { 13 } } \\ { a _ { 21 } } &amp;
+{ a _ { 22 } } &amp; { a _ { 23 } } \\ { a _ { 31 } } &amp; { a _ { 32 }
+} &amp; { a _ { 33 } } \end{array} \right] = \left[ \begin{array} { c c
+c } { l _ { 11 } } &amp; { 0 } &amp; { 0 } \\ { l _ { 21 } } &amp; { l _
+{ 22 } } &amp; { 0 } \\ { l _ { 31 } } &amp; { l _ { 32 } } &amp; { l _
+{ 33 } } \end{array} \right] \left[ \begin{array} { c c c } { u _ { 11 }
+} &amp; { u _ { 12 } } &amp; { u _ { 13 } } \\ { 0 } &amp; { u _ { 22 }
+} &amp; { u _ { 23 } } \\ { 0 } &amp; { 0 } &amp; { u _ { 33 } }
+\end{array} \right]\]</span></p>
+<p>分解过程：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/matrix/2017-03-29-20-13-15.png" /></p>
+<p>声明：l11表示L矩阵的第一行第一列的元素，U1i表示L矩阵第一行每一列（需要遍历的列）的元素</p>
 <ol type="1">
-<li><code>!=</code> 将不使用索引. 记住, 索引只能告诉你什么存在于表中,
-而不能告诉你什么不存在于表中.</li>
-<li><code>¦ ¦</code>是字符连接函数. 就象其他函数那样, 停用了索引.</li>
-<li><code>+</code>是数学函数. 就象其他数学函数那样, 停用了索引.</li>
-<li>相同的索引列不能互相比较,这将会启用全表扫描.</li>
-</ol></li>
-<li><ol type="a">
-<li><p>如果检索数据量超过30%的表中记录数.使用索引将没有显著的效率提高.</p></li>
-<li><p>在特定情况下, 使用索引也许会比全表扫描慢,
-但这是同一个数量级上的区别.
-而通常情况下,使用索引比全表扫描要块几倍乃至几千倍!</p></li>
+<li>初始化L、U矩阵。L矩阵下三角矩阵，对角元素全为1；U矩阵为上三角矩阵。</li>
+<li>计算U矩阵的第一行，对应A矩阵的第一行元素，不需要做任何操作</li>
+<li>计算L矩阵的第一列（除去第一行元素），对应A矩阵的第一列元素，每个元素除以u11</li>
+<li>计算U矩阵的第二行（除去第一列元素），对应A矩阵的第二行元素，每个元素减去l21*u1i</li>
+<li>计算L矩阵的第二列（除去前两行元素），对应A矩阵的第二列元素，每个元素先减去l31*u12，再除以u22</li>
+<li>……</li>
+</ol>
+<p>总结公式如下：</p>
+<p>L矩阵对角元素为1。</p>
+<p>第一步的公式：</p>
+<p><span class="math display">\[\begin{array}{lll}
+a_{1j}=(1,0,0,\cdots,0)\cdot(u_{1j},u_{2j},\cdots,u_{ij}).T=u_{1j} \\
+a_{i1}=(l_{i1},
+l_{i1},\cdots,l_{i1})\cdot(u_{11},0,\cdots,0).T=l_{i1}\cdot u_{11}
+\\u_{1j}=a_{1j}  \\ l_{i1}=\frac{a_{i1}}{u_{11}} \\j=1,2,\cdots,n
+\\i=2,3,\cdots,n\end{array}\]</span></p>
+<p>第r步的公式：</p>
+<p><span
+class="math display">\[\begin{array}{rl}  a_{rj}&amp;=(l_{r1},l_{r2},...,l_{rr-1})\cdot(u_{1j},u_{2j},...,u_{jj}).T
+\\&amp;=\sum\limits_{k=1}^{r-1}l_{rk}u_{kj}+u_{rj}  \end{array}\]</span></p>
+<p><span class="math display">\[\begin{array}{lll}
+u_{rj}=a_{rj}-\sum\limits_{k=1}^{r-1}l_{rk}u_{kj} \\
+l_{ir}=\frac{a_{ir}-\sum\limits_{k=1}^{r-1}l_{ik}u_{kr}}{u_{rr}}
+\end{array}\]</span></p>
+<p><span class="math display">\[\begin{array}{lll} j=r,r+1,\cdots,n \\
+r=2,3,\cdots,n-1 \\ i=r+1,\cdots,n\end{array}\]</span></p>
+<h3 id="qr分解">3. QR分解</h3>
+<p><a
+href="https://wenku.baidu.com/view/c2e34678168884868762d6f9.html">参考链接</a></p>
+<p>把矩阵分解成一个半正交矩阵与一个上三角矩阵的积。</p>
+<p>形式如:</p>
+<p><span class="math display">\[A=QR\]</span></p>
+<p>分解定理：任意一个满秩实（复）矩阵A，都可唯一地分解A=QR,其中Q为正交（酉）矩阵，R是具有对角元的上三角矩阵。</p>
+<h4 id="三种实现方式">三种实现方式</h4>
+<ol type="1">
+<li><p>Gram-Schmidt正交化</p>
+<p>过程:</p>
+<ol type="1">
+<li><p>取出矩阵A中的每一列命名为x_col，它们之间线性无关</p></li>
+<li><p>将每一个x_col正交化得到y_col</p></li>
+<li><p>再将y_col进行单位化，即转换为e_col。这里每个e_col为矩阵Q中的每一列。</p></li>
+<li><p>将第二步中，先转换为y_col的正交化得到x_col，再转换为e_col正交化得到x_col。这里每个e_col的系数，对应的就是矩阵R中的每一列</p></li>
 </ol></li>
-<li><p>避免使用耗费资源的操作</p>
-<p>带有DISTINCT,UNION,MINUS,INTERSECT,ORDER
-BY的SQL语句会启动SQL引擎执行耗费资源的排序(SORT)功能.
-DISTINCT需要一次排序操作, 而其他的至少需要执行两次排序. 通常, 带有UNION,
-MINUS , INTERSECT的SQL语句都可以用其他方式重写.
-如果你的数据库的SORT_AREA_SIZE调配得好, 使用UNION , MINUS,
-INTERSECT也是可以考虑的, 毕竟它们的可读性很强。</p></li>
-<li><p>优化GROUP BY</p>
-<p>提高GROUP BY 语句的效率, 可以通过将不需要的记录在GROUP BY
-之前过滤掉.下面两个查询返回相同结果但第二个明显就快了许多.</p></li>
+<li><p>Householder变换</p>
+<p><a href="https://www.bbsmax.com/A/gVdnKqK1zW/">参考链接</a></p>
+<p>outer函数</p>
+<ul>
+<li><p>通常有两个输入向量，假设为向量a、b</p>
+<p><span class="math display">\[a = (1, 2, 3) , b = (-1, -2,
+-3)\]</span></p></li>
+<li><p>那么，outer函数运算过程如</p></li>
+</ul>
+<p><span class="math display">\[outer(a, b) = \left[ \begin{array}{ccc}
+1*-1 &amp; 1*-2 &amp; 1*-3 \\ 2*1 &amp; 2*-2 &amp; 2*-3 \\ 3*-1 &amp;
+3*-2 &amp; 3*-3 \end{array} \right]\]</span></p>
+<p>过程:</p>
+<ol type="1">
+<li>初始化矩阵Q为单位矩阵(rxr)，r为矩阵A的行数。初始化矩阵R为矩阵A</li>
+<li>取R中的下三角矩阵，每次取一列命名为x（不包括最后一列）。e初始化为形如x的零向量</li>
+<li>e中的第一个元素为向量x的二范数</li>
+<li>这里得到v，范数取2</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br><span class="line"><span class="keyword">HAVING</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">WHERE</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br></pre></td></tr></table></figure>
-<pre><code>​</code></pre>
+<p><span class="math display">\[v = \frac{x - e}{||x-e||}\]</span></p>
+<ol start="5" type="1">
+<li>初始化一个单位矩阵命名为Q_cnt，取Q_cnt的下三角矩阵，每次取一列在原有的基础上减去2*outer(v,
+v)。（不包括最后一列）</li>
+<li>更新R,Q</li>
+</ol>
+<p><span class="math display">\[R = Q\_cnt * R \]</span></p>
+<p><span class="math display">\[Q = Q * Q\_cnt\]</span></p></li>
+<li><p>Givens变换</p>
+<p><a
+href="http://www.voidcn.com/article/p-vmnqitql-xa.html">参考链接</a></p>
+<p>有待补充</p></li>
+</ol>
+]]></content>
+      <categories>
+        <category>Algorithm</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>数学</tag>
+        <tag>矩阵分解</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>风格迁移转换的两种实现</title>
+    <url>/hexo_blog/2017/08/14/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB%E8%BD%AC%E6%8D%A2%E7%9A%84%E4%B8%A4%E7%A7%8D%E5%AE%9E%E7%8E%B0/</url>
+    <content><![CDATA[<p>本文介绍了如何在Ubuntu16.04环境下搭建caffe和tensorflow，并利用caffe和tensorflow实现了风格转换。同时，介绍了如何训练新的风格模型。</p>
+<span id="more"></span>
+<p><strong>style-transfer</strong></p>
+<p>这个主要是利用caffe+model+code直接实现任意风格的转换。</p>
+<p>最后实现需借助</p>
+<ul>
+<li>一张风格图片</li>
+<li>待转换风格的目标图片</li>
+<li>训练模型</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br></pre></td></tr></table></figure>
+<p>优势：</p>
+<ul>
+<li>可以实现任意风格的转换</li>
+<li>可拓展性强 缺点：</li>
+<li>耗时长（真的很长，如果用CPU的话）</li>
+</ul>
+<p><strong>fast-neural-style-tensorflow</strong></p>
+<p>风格快速迁移转换</p>
+<p>顾名思义，这个比前者能够更快的进行风格转换，速度因机器而异，不过效率比前者确确实实提高了很多倍</p>
+<p>最后实现需借助</p>
+<ul>
+<li>一个训练好的风格模型</li>
+<li>待转换的风格图片</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding</span></span><br><span class="line"> python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br></pre></td></tr></table></figure>
+<p>优势：</p>
+<ul>
+<li>速度相对来说很快</li>
+<li>环境搭建相对来说更容易 缺点：</li>
+<li>可拓展性更弱</li>
+<li>只能转换固定几种风格</li>
+<li>训练新风格模型时间长（比前者生成时间更长）</li>
+</ul>
+<h2 id="style-transfer的实现">style-transfer的实现</h2>
+<p><strong>环境介绍</strong></p>
+<p>Ubuntu16.04 + CPU + python2.7 + caffe</p>
+<h3 id="搭建caffe环境">搭建caffe环境</h3>
+<h4 id="介绍">介绍</h4>
+<p>Caffe是一个清晰而高效的深度学习框架。Caffe是纯粹的C++/CUDA架构，支持命令行、Python和MATLAB接口；可以在CPU和GPU直接无缝切换。</p>
+<p>优势：</p>
+<ul>
+<li>上手快：模型与相应优化都是以文本形式而非代码形式给出。
+Caffe给出了模型的定义、最优化设置以及预训练的权重，方便立即上手。</li>
+<li>速度快：能够运行最棒的模型与海量的数据。
+Caffe与cuDNN结合使用，测试AlexNet模型，在K40上处理每张图片只需要1.17ms.</li>
+<li>模块化：方便扩展到新的任务和设置上。
+可以使用Caffe提供的各层类型来定义自己的模型。</li>
+<li>开放性：公开的代码和参考模型用于再现。</li>
+<li>社区好：可以通过BSD-2参与开发与讨论。</li>
+</ul>
+<h4 id="安装依赖">安装依赖</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># caffe 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler</span><br><span class="line"><span class="built_in">sudo</span> apt-get install --no-install-recommends libboost-all-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libatlas-base-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libhdf5-serial-dev</span><br><span class="line"><span class="comment"># Python 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libgflags-dev libgoogle-glog-dev liblmdb-dev</span><br></pre></td></tr></table></figure>
+<h4
+id="安装caffe可利用virtualenv新建一个纯净环境">安装caffe（可利用virtualenv新建一个纯净环境）</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 我们需要将caffe的源码下载下来，所以需要git</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install git</span><br><span class="line"><span class="comment"># 下载代码</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/BVLC/caffe.git</span><br><span class="line"><span class="comment"># 下载完成后，进入caffe文件夹</span></span><br><span class="line"><span class="built_in">cd</span> caffe</span><br><span class="line"><span class="comment"># 安装caffe的python依赖</span></span><br><span class="line"><span class="keyword">for</span> req <span class="keyword">in</span> $(<span class="built_in">cat</span> requirements.txt); <span class="keyword">do</span> pip install <span class="variable">$req</span>; <span class="keyword">done</span></span><br></pre></td></tr></table></figure>
+<h4 id="配置caffe">配置caffe</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先确保在caffe文件夹中，然后修改配置文件</span></span><br><span class="line"><span class="comment"># 进行备份</span></span><br><span class="line"><span class="built_in">cp</span> Makefile.config.example Makefile.config</span><br><span class="line"><span class="comment"># 进行编辑</span></span><br><span class="line">vim Makefile.config</span><br></pre></td></tr></table></figure>
+<ul>
+<li>因为CPU MODE, 所以在CPU_ONLY := 1前面的#要去掉</li>
+<li>两个路径要改成这样:(添加后面的两个hdf5的路径,
+否则编译时报hdf5错误)</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Whatever else you find you need goes here.</span></span><br><span class="line">INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial</span><br><span class="line">LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial</span><br></pre></td></tr></table></figure>
+<p>修改完成之后，保存退出即可</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行编译</span></span><br><span class="line">make pycaffe</span><br><span class="line">make all</span><br><span class="line">make <span class="built_in">test</span></span><br><span class="line">make runtest</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译结束后，需要设置环境变量</span></span><br><span class="line"><span class="comment"># 首先确保已经进入到caffe文件夹中，之后查看当前路径</span></span><br><span class="line"><span class="built_in">pwd</span></span><br><span class="line"><span class="comment"># 假设输出结果是xxx，那么将xxx复制下来，黏贴至下面xxx的位置</span></span><br><span class="line"><span class="built_in">export</span> PYTHONPATH=xxx/python:<span class="variable">$PYTHONPATH</span></span><br></pre></td></tr></table></figure>
+<h4 id="测试">测试</h4>
+<p>一般来说，如果运气好的话，到这里基本就算安装完成，但我们还是要测试一下</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进入python环境</span></span><br><span class="line">python</span><br><span class="line"><span class="comment"># 导入caffe模块</span></span><br><span class="line">&gt;caffe</span><br><span class="line"><span class="comment"># 如果此处没有报任何错误，恭喜你安装成功</span></span><br></pre></td></tr></table></figure>
+<h4 id="部分报错解决方案">部分报错解决方案</h4>
+<p><strong>最好的方案莫过于，复制报错代码，Google一下 </strong>
+下面提供几种，我安装时的几处坑</p>
+<ul>
+<li>设置环境变量，注意一定是在caffe文件夹下，路径是caffe/python的绝对路径</li>
+<li>安装依赖报错，这个没什么好讲，直接google吧</li>
+<li>make的时候报错，想想看是否编辑了Makefile.config。之后make
+clean一下，再重新编译一下</li>
+<li>CPU_ONLY这个选项为1时，需要修改一下<code>caffe/examples/mnist/lenet_solver.prototxt</code>，将其中的<code>solver_mode:GPU</code>改为CPU</li>
+</ul>
+<p><a
+href="http://blog.csdn.net/sinat_26917383/article/details/53502719">参考文章</a></p>
+<h3 id="实现前准备">实现前准备</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认已安装git，未安装请sudo apt get install git</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/fzliu/style-transfer</span><br><span class="line"></span><br><span class="line"><span class="comment"># pycaffe环境布置</span></span><br><span class="line"><span class="built_in">sudo</span> pip install progressbar</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载训练模型，这里推荐vgg16</span></span><br><span class="line"><span class="comment"># 方法一，使用scripts/ 下的download_models.sh这个方法可能很慢。。。。</span></span><br><span class="line">bash scripts/download_models.sh vgg16</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">wget http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_16_layers.caffemodel</span><br></pre></td></tr></table></figure>
+<p>使用方法二下载的model需要放在./model/vgg16下，使用方法一下载的已经默认放在那</p>
+<h3 id="实现">实现</h3>
+<p>确保在源码文件夹中</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子，model_name直接写vgg16或其他，不需要加具体路径</span></span><br><span class="line">python style.py -s images/style/starry_night.jpg -c images/content/nanjing.jpg -m vgg16 -g -1</span><br></pre></td></tr></table></figure>
+<p>提示：CPU很慢，真的很慢，估计要等一会才会有时间显示出来</p>
+<p>参数解析：</p>
+<ul>
+<li>-s, 风格图位置；</li>
+<li>-c, 内容图位置；</li>
+<li>-m, 模型位置；</li>
+<li>-g, 什么模式，-1为CPU，0为单个GPU，1为两个GPU。</li>
+</ul>
+<h3 id="调整参数">调整参数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保在源码文件夹下,修改style.py配置文件</span></span><br><span class="line">vim style.py</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改文件大小，改为1024</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-l&quot;</span>, <span class="string">&quot;--length&quot;</span>, default=1024, <span class="built_in">type</span>=<span class="built_in">float</span>, required=False, <span class="built_in">help</span>=<span class="string">&quot;maximum image length&quot;</span>)</span><br><span class="line">def transfer_style(self, img_style, img_content, length=1024, ratio=1e5,</span><br><span class="line">n_iter=512, init=<span class="string">&quot;-1&quot;</span>, verbose=False, callback=None)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改迭代次数，个人认为400就差不多了，可自行修改；50次迭代之后改变的就是背景纹理，按需更改</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-n&quot;</span>, <span class="string">&quot;--num-iters&quot;</span>, default=400, <span class="built_in">type</span>=int, required=False, <span class="built_in">help</span>=<span class="string">&quot;L-BFGS iterations&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 还有很多其他可以修改的地方，在此就不一一赘述了</span></span><br></pre></td></tr></table></figure>
+<h2
+id="fast-neural-style-tensorflow的实现">fast-neural-style-tensorflow的实现</h2>
+<p><strong>环境介绍 </strong></p>
+<p>Ubuntu16.04 + CPU + python2.7 + tensorflow 1.0</p>
+<h3 id="搭建tensorflow环境">搭建tensorflow环境</h3>
+<p>环境说明：Ubuntu16.04 注：tensorflow只支持64位系统</p>
+<h4 id="安装前准备">安装前准备</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保有pip工具，一般自带，没有的话请按下面这条命令进行安装</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-pip python-dev</span><br></pre></td></tr></table></figure>
+<h4 id="cpu安装">CPU安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip installl tensorflow</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install tensorflow</span><br></pre></td></tr></table></figure>
+<h4 id="gpu安装">GPU安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装地址请根据自己需求进行声明环境变量</span></span><br><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br></pre></td></tr></table></figure>
+<p><a
+href="https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/1-2-install/">参考文章</a></p>
+<h4 id="源码安装">源码安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载最新的tensorflow项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/tensorflow/tensorflow</span><br><span class="line"><span class="comment"># 进入tensorflow</span></span><br><span class="line"><span class="built_in">cd</span> tensorflow</span><br><span class="line"><span class="comment"># 切换到需要安装的版本分支，以1.0为栗子</span></span><br><span class="line">git checkout r1.0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装bazel</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8&quot;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> /etc/apt/sources.list.d/bazel.list</span><br><span class="line"></span><br><span class="line">curl https://bazel.build/bazel-release.pub.gpg | <span class="built_in">sudo</span> apt-key add -</span><br><span class="line"></span><br><span class="line"><span class="built_in">sudo</span> apt-get update &amp;&amp; <span class="built_in">sudo</span> apt-get install bazel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装依赖</span></span><br><span class="line"><span class="comment"># Python 2</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-numpy python-dev python-pip python-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python 3</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python3-numpy python3-dev python3-pip python3-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行编译，确保在tensorflow文件夹中</span></span><br><span class="line">./configure</span><br><span class="line"><span class="comment"># 提示信息可以一路回车</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译CPU版本</span></span><br><span class="line">bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"><span class="comment"># 编译GPU版本</span></span><br><span class="line">bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译生成的whl包</span></span><br><span class="line">bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装生成的包</span></span><br><span class="line"><span class="built_in">sudo</span> pip install /tmp/tensorflow_pkg/tensorflow*.whl</span><br></pre></td></tr></table></figure>
+<p><a
+href="https://tensorflow.feisky.xyz/install/src.html">参考文章</a></p>
+<h4 id="测试-1">测试</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 打开对应版本的python终端</span></span><br><span class="line">&gt;import tensorflow</span><br><span class="line"><span class="comment"># 未报错即安装成功</span></span><br></pre></td></tr></table></figure>
+<h3 id="实现前准备-1">实现前准备</h3>
+<p>下载已经训练好的模型</p>
+<p><a
+href="https://pan.baidu.com/s/1i4GTS4d#list/path=%2F">百度云盘链接</a></p>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 安装pyyaml</span><br><span class="line">sudo pip install pyyaml</span><br></pre></td></tr></table></figure>
+<h3 id="实现-1">实现</h3>
+<p>确保在源码文件夹中</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子,假设wave.ckpt-done放在了源码文件夹中的models文件夹中</span></span><br><span class="line">python eval.py --model_file models/wave.ckpt-done --image_file img/test.jpg</span><br><span class="line"><span class="comment"># 默认生成位置在generated，默认生成文件名称为res.jpg。可自行在eval.py中更改</span></span><br></pre></td></tr></table></figure>
+<h3 id="训练新的风格模型">训练新的风格模型</h3>
+<p><strong>下载</strong></p>
+<p><a
+href="http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz">vgg16训练模型</a>(500多M)</p>
+<p><a
+href="http://msvocds.blob.core.windows.net/coco2014/train2014.zip">训练数据集</a>(12.6G)</p>
+<p>或者使用wget命令下载</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># vgg16训练模型</span></span><br><span class="line">wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练数据集</span></span><br><span class="line">wget http://msvocds.blob.core.windows.net/coco2014/train2014.zip</span><br></pre></td></tr></table></figure>
+<p><strong>移动模型到对应目录中</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码文件夹中新建文件夹</span></span><br><span class="line"><span class="built_in">mkdir</span> pretrained</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载完成之后需要进行解压</span></span><br><span class="line">tar -zxvf vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 移动文件夹到pretrained</span></span><br><span class="line"><span class="built_in">cp</span> &lt;path of vgg16&gt; &lt;pretrained&gt;</span><br></pre></td></tr></table></figure>
+<p><strong>训练新的模型</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压数据集</span></span><br><span class="line">unzip train2014.zip</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入数据集</span></span><br><span class="line"><span class="built_in">cd</span> train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立软链接，方便训练命令的输入</span></span><br><span class="line"><span class="built_in">ln</span> -s &lt;train2014的绝对路径&gt; train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入源码文件夹中</span></span><br><span class="line"><span class="comment"># 开始训练, wave.yml是作者预先配置好的文件，如果训练自己的模型，需要自己新写一份yml文件</span></span><br><span class="line">python train.py -c conf/wave.yml</span><br></pre></td></tr></table></figure>
+<h2 id="总结">总结</h2>
+<p>如果只是单纯的想玩玩风格转换的话，可以试试fast-neural-style-tensorflow，不过训练新模型还是不建议（特殊需要除外），耗时耗力</p>
+<p>如果想锻炼自己，从零开始学习，可以从读论文开始，然后试试style-transfer代码</p>
+<p><a
+href="http://blog.csdn.net/sinat_26917383/article/details/53978519">style-transfer
+参考文章</a></p>
+<p><a href="https://github.com/fzliu/style-transfer">style-transfer
+代码</a></p>
+<p><a
+href="https://github.com/hzy46/fast-neural-style-tensorflow/blob/master/README.md#use-trained-models">fast-neural-style-tensorflow
+代码</a></p>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Environment</category>
       </categories>
       <tags>
-        <tag>数据库</tag>
-        <tag>SQL</tag>
+        <tag>Python</tag>
+        <tag>Caffe</tag>
+        <tag>Tensorflow</tag>
+        <tag>风格迁移</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅰ</title>
-    <url>/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+    <title>线性代数中各种矩阵的简介</title>
+    <url>/hexo_blog/2019/05/13/%E6%95%B0%E5%AD%A6/%E7%BA%BF%E4%BB%A3%E7%9F%A9%E9%98%B5/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了线性代数的矩阵，仅作笔记。</p>
 <span id="more"></span>
-<h2 id="写在开头">写在开头</h2>
-<p>在这里主要是对NLP的相关知识做一个整理和对NLTK模块的介绍，书中Python基础内容在这里不做介绍，如书中有我认为值得介绍的Python写法，我会进行说明。</p>
-<p>由于书中nltk是老版本的问题，语法上存在一些变动，在这里也会进行修正。可能由于书是译本的原因（当然也可能原作者自己的失误），部分代码有些错误，在这里我也进行了校正。如果您发现了我的文章中有需要改正或改进的地方，欢迎在评论区提出。</p>
-<p>最后十分感谢原作者的贡献和译者的翻译，强烈推荐读者亲自阅读此书。</p>
-<p>nltk版本是3.2.3，<a href="http://www.nltk.org/">官网</a></p>
-<p>python版本3.6.1</p>
-<p>数据、PDF和一些重要的代码已经放在了Github上了。<a
-href="https://github.com/wnma3mz/Nltk_Study">github地址</a></p>
-<h2 id="第一章语言处理与python">第一章：语言处理与Python</h2>
-<p>NLTK库的安装，在这里不做介绍，强烈建议直接使用Anaconda环境。<a
-href="https://www.anaconda.com/download/">官网下载链接</a>。本文是直接使用了Anaconda环境。</p>
-<p>数据的获取，这里nltk有官方提供的文本数据，可以直接使用<code>nltk.download()</code>打开图形界面，下载语料集<code>book</code>。由于使用这个方法下载速度比较慢的原因，我在这里Github上提供了数据集<code>nltk_data</code>，下载之后，移动到<code>nltk.download()</code>原本的下载目录下，之后再运行<code>nltk.download()</code>就不需要下载数据集而是直接解压数据集，速度会快上很多。<a
-href="https://github.com/wnma3mz/Nltk_Study">下载链接</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk模块</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 导入基本语料集(不需要额外下载)，包含text1到text9变量，可以直接输出这些变量</span></span><br><span class="line"><span class="keyword">from</span> nltk.book <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本。这里表示找到&quot;monstrous&quot;所包含的句子，并且输出上下文</span></span><br><span class="line">text1.concordance(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本出现在相似的上下文中</span></span><br><span class="line">text1.similar(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索两个及两个以上共同词的上下文</span></span><br><span class="line">text2.common_contexts([<span class="string">&quot;monstrous&quot;</span>, <span class="string">&quot;very&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画一张离散图表示这些词出现在文本中的位置，输出见下图</span></span><br><span class="line">text4.dispersion_plot([<span class="string">&quot;citizens&quot;</span>, <span class="string">&quot;democracy&quot;</span>, <span class="string">&quot;freedom&quot;</span>, <span class="string">&quot;duties&quot;</span>, <span class="string">&quot;America&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于文本，随机生成一些文本</span></span><br><span class="line">text3.generate()</span><br></pre></td></tr></table></figure>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-1.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 有序字典，按词频从高到低排序</span></span><br><span class="line">fdist1 = FreqDist(text1)</span><br><span class="line"><span class="comment"># 选出词频最高的50个词</span></span><br><span class="line">fdist1.keys()[:<span class="number">50</span>]</span><br><span class="line"><span class="comment"># 某个词出现的频数</span></span><br><span class="line">fdist1[<span class="string">&#x27;whale&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># text1中词频最高的50个单词，进行绘图，输出见下图</span></span><br><span class="line">fdist1.plot(<span class="number">50</span>, cumulative=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># text1中只出现过一次的单词</span></span><br><span class="line">fdist1.hapaxes()</span><br></pre></td></tr></table></figure>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-3.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 词语搭配，双连词(bigrams)</span></span><br><span class="line">nltk.bigrams([<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>, <span class="string">&#x27;done&#x27;</span>])</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[(<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>),(<span class="string">&#x27;is&#x27;</span>,<span class="string">&#x27;said&#x27;</span>),(<span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>), (<span class="string">&#x27;than&#x27;</span>,<span class="string">&#x27;done&#x27;</span>)]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文本中单个词的频率预期得到的更频繁出现的双连词</span></span><br><span class="line">text4.collocations()</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 与nltk自带的聊天机器人系统对话</span></span><br><span class="line"><span class="comment"># 导入模块</span></span><br><span class="line"><span class="keyword">from</span> nltk.chat <span class="keyword">import</span> chatbots</span><br><span class="line"><span class="comment"># 选择一个聊天机器人并开始对话</span></span><br><span class="line">chatbots()</span><br></pre></td></tr></table></figure>
-<p>补充部分：</p>
-<table>
-<colgroup>
-<col style="width: 7%" />
-<col style="width: 92%" />
-</colgroup>
-<thead>
-<tr>
-<th style="text-align: center;">text</th>
-<th style="text-align: center;">文本名</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align: center;">text1</td>
-<td style="text-align: center;">《白鲸记》（Moby Dick by Herman Melville
-1851）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text2</td>
-<td style="text-align: center;">《理智与情感》（Sense and Sensibility by
-Jane Austen 1811）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text3</td>
-<td style="text-align: center;">《创世纪》（The Book of Genesis）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text4</td>
-<td style="text-align: center;">《就职演说语料库》（Inaugural Address
-Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text5</td>
-<td style="text-align: center;">《NPS聊天语料库》（Chat Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text6</td>
-<td style="text-align: center;">《巨蟒与圣杯》（Monty Python and the
-Holy Grail）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text7</td>
-<td style="text-align: center;">《华尔街日报》（Wall Street
-Journal）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text8</td>
-<td style="text-align: center;">《个人文集》（Personals Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text9</td>
-<td style="text-align: center;">《周四的男人》（The Man Who Was Thursday
-by G . K . Chestert）</td>
-</tr>
-</tbody>
-</table>
-<p>概念补充：</p>
-<ul>
-<li>词类型：一个词在一个文本中独一无二的出现形式或拼写。也就是说，这个词在词汇表中是唯一的。</li>
-<li>频率分布：</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-2.png" /></p>
-<ul>
-<li>FreqDist的API</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-4.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-5.png" /></p>
-<ul>
-<li>词意消歧：需要算出特定上下文中的词被赋予的是哪个意思。单词可能存在相同/相近的含义，此时需要根据上下文来推断单词在此情景下的含义。</li>
-<li>指代消解（anaphora resolution）：确定代词或名词短语指的是什么</li>
-<li>先行词：代词可能代表的对象</li>
-<li>语义角色标注（semantic role
-labeling）：确定名词短语如何与动词相关联（如施事，受事，工具等）</li>
-<li>自动生成语言：需要解决自动语言理解。弄清楚词的含义、动作的主语以及代词的先行词是理解句子含义的步骤</li>
-<li>机器翻译（MT）：难点，一个给定的词可能存在几种不同的解释。</li>
-<li>文本对齐：根据一个网站发布的多种语言版本，来自动配对组成句子</li>
-<li>人机对话系统：图灵测试。</li>
-<li>文本含义识别（Recognizing Textual Entailment 简称
-RTE）：根据假定的一些条件，来推断给出的一句话是否正确。</li>
-</ul>
-<h2
-id="第二章获得文本语料和词汇资源">第二章：获得文本语料和词汇资源</h2>
-<p><a href="http://www.gutenberg.org/">古腾堡语料库</a></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-1.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk的古腾堡语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> gutenberg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看这个语料库中的所有txt文件名</span></span><br><span class="line">gutenberg.fileids()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中其中的一个</span></span><br><span class="line">emma = gutenberg.words(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 查看它包含的词汇个数</span></span><br><span class="line"><span class="built_in">len</span>(emma)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果要使用第一章中concordance()这样的命令，就必须要将数据放到nltk.Text对象中</span></span><br><span class="line">emma = nltk.Text(emma)</span><br><span class="line">emma.concordance(<span class="string">&quot;suprprize&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用words获取的是经过分隔成标识符的文本。使用sents函数则可以获取以句子进行划分后的数据。具体区别可以运行看看，请读者自行对比。</span></span><br><span class="line">emma_sents = gutenberg.sents(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取原始文本。一个字符串</span></span><br><span class="line">emma_raw = gutenberg.raw(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网络和聊天文本</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> webtext</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> nps_chat</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库中所有的类别</span></span><br><span class="line">brown.categories()</span><br><span class="line"><span class="comment"># words函数根据类别选择数据</span></span><br><span class="line">brown.words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">[<span class="string">&#x27;The&#x27;</span>, <span class="string">&#x27;Fulton&#x27;</span>, <span class="string">&#x27;County&#x27;</span>, <span class="string">&#x27;Grand&#x27;</span>, <span class="string">&#x27;Jury&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># words函数根据文件名选择数据</span></span><br><span class="line">brown.words(fileids=[<span class="string">&#x27;cg22&#x27;</span>])</span><br><span class="line">[<span class="string">&#x27;Does&#x27;</span>, <span class="string">&#x27;our&#x27;</span>, <span class="string">&#x27;society&#x27;</span>, <span class="string">&#x27;have&#x27;</span>, <span class="string">&#x27;a&#x27;</span>, <span class="string">&#x27;runaway&#x27;</span>, <span class="string">&#x27;,&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># sents函数根据类别选择数据</span></span><br><span class="line">brown.sents(categories=[<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;editorial&#x27;</span>, <span class="string">&#x27;reviews&#x27;</span>])</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="comment"># 带条件的频率分布函数</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 取出brown语料库中所有类别中的所有词。每个类别对应该类别下的所有词。(类别，所有词)</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (genre, word)</span><br><span class="line">        <span class="keyword">for</span> genre <span class="keyword">in</span> brown.categories()</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> brown.words(categories=genre))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定一部分文本类别(genres)</span></span><br><span class="line">genres = [<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;religion&#x27;</span>, <span class="string">&#x27;hobbies&#x27;</span>, <span class="string">&#x27;science_fiction&#x27;</span>, <span class="string">&#x27;romance&#x27;</span>, <span class="string">&#x27;humor&#x27;</span>]</span><br><span class="line"><span class="comment"># 指定一些情态动词(modals)</span></span><br><span class="line">modals = [<span class="string">&#x27;can&#x27;</span>, <span class="string">&#x27;could&#x27;</span>, <span class="string">&#x27;may&#x27;</span>, <span class="string">&#x27;might&#x27;</span>, <span class="string">&#x27;must&#x27;</span>, <span class="string">&#x27;will&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出一张表格。genres与modals的对应数量关系</span></span><br><span class="line">cfd.tabulate(conditions=genres, samples=modals)</span><br><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">                  can could   may might  must  will</span></span><br><span class="line"><span class="string">           news    93    86    66    38    50   389</span></span><br><span class="line"><span class="string">       religion    82    59    78    12    54    71</span></span><br><span class="line"><span class="string">        hobbies   268    58   131    22    83   264</span></span><br><span class="line"><span class="string">science_fiction    16    49     4    12     8    16</span></span><br><span class="line"><span class="string">        romance    74   193    11    51    45    43</span></span><br><span class="line"><span class="string">          humor    16    30     8     8     9    13</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br></pre></td></tr></table></figure>
-<p>通过输出，我们很容易发现，在<code>news</code>类别里面用的最多的情态动词是<code>will</code>。当然，还有一些结论就由读者自行发现了。</p>
-<p>关于其他语料库的信息挖掘，在这里就不赘述了。书中也只是浅显的用了之前用过的一些方法。下面展示一些我认为比较有意思的代码及输出。</p>
-<p>官网提供了如何访问NLTK语料库的其他例子，<a
-href="http://www.nltk.org/howto/corpus.html">链接</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> inaugural</span><br><span class="line"></span><br><span class="line"><span class="comment"># 先选出inaugural语料库中的所有字段的对应所有词汇，再统计出文本中以`america`和`citizen`开头的词在每个年份（filed）出现次数。（`america`或`citizen`, 年份）</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (target, fileid[:<span class="number">4</span>])</span><br><span class="line">        <span class="keyword">for</span> fileid <span class="keyword">in</span> inaugural.fileids()</span><br><span class="line">        <span class="keyword">for</span> w <span class="keyword">in</span> inaugural.words(fileid)</span><br><span class="line">        <span class="keyword">for</span> target <span class="keyword">in</span> [<span class="string">&#x27;america&#x27;</span>, <span class="string">&#x27;citizen&#x27;</span>]</span><br><span class="line">        <span class="keyword">if</span> w.lower().startswith(target))</span><br><span class="line"><span class="comment"># 绘图</span></span><br><span class="line">cfd.plot()</span><br></pre></td></tr></table></figure>
-<p>以<code>america</code>或<code>citizen</code>开始的词随时间（年份）的演变趋势</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-2.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># udhr语料库中不同语言版本的字长差异（彩色图）</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> udhr</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选取比较的语言</span></span><br><span class="line">languages = [<span class="string">&#x27;Chickasaw&#x27;</span>, <span class="string">&#x27;English&#x27;</span>, <span class="string">&#x27;German_Deutsch&#x27;</span>, <span class="string">&#x27;Greenlandic_Inuktikut&#x27;</span>, <span class="string">&#x27;Hungarian_Magyar&#x27;</span>, <span class="string">&#x27;Ibibio_Efik&#x27;</span>]</span><br><span class="line"><span class="comment"># 提取不同语言的数据</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (lang, <span class="built_in">len</span>(word))</span><br><span class="line">        <span class="keyword">for</span> lang <span class="keyword">in</span> languages</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> udhr.words(lang + <span class="string">&#x27;-Latin1&#x27;</span>))</span><br><span class="line"></span><br><span class="line">cfd.plot(cumulative=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
-<p>累积字长分布： “世界人权宣言”的6个翻译版本 <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-3.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 载入自己的语料库，文本文件</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> PlaintextCorpusReader</span><br><span class="line"><span class="comment"># 设置语料库的路径</span></span><br><span class="line">corpus_root = <span class="string">&#x27;/usr/share/dict&#x27;</span></span><br><span class="line"><span class="comment"># 解析路径下的所有文件</span></span><br><span class="line">wordlists = PlaintextCorpusReader(corpus_root, <span class="string">&#x27;.*&#x27;</span>)</span><br><span class="line"><span class="comment"># 语料库中所有的文件名</span></span><br><span class="line">wordlists.fileids()</span><br><span class="line"><span class="comment"># 语料库中`connectives`中的单词</span></span><br><span class="line">wordlists.words(<span class="string">&#x27;connectives&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 本地个人的语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> BracketParseCorpusReader</span><br><span class="line">corpus_root = <span class="string">r&quot;C:\corpora\penntreebank\parsed\mrg\wsj&quot;</span></span><br><span class="line"><span class="comment"># 使用正则匹配</span></span><br><span class="line">file_pattern = <span class="string">r&quot;.*/wsj_.*\.mrg&quot;</span></span><br><span class="line">ptb = BracketParseCorpusReader(corpus_root, file_pattern)</span><br><span class="line">ptb.fileids()</span><br></pre></td></tr></table></figure>
-<p>双连词运用 <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 依次连续输出最有可能跟在上一个词后面的词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">generate_model</span>(<span class="params">cfdist, word, num=<span class="number">15</span></span>):</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num):</span><br><span class="line">        <span class="built_in">print</span>(word)</span><br><span class="line">        word = cfdist[word].<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 选中语料库，提取文本</span></span><br><span class="line">text = nltk.corpus.genesis.words(<span class="string">&#x27;english-kjv.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 生成双连词</span></span><br><span class="line">bigrams = nltk.bigrams(text)</span><br><span class="line"><span class="comment"># 创建双连词频数对象</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(bigrams)</span><br><span class="line"><span class="comment"># 找出最有可能在`living`后面的单词。</span></span><br><span class="line"><span class="built_in">print</span>(cfd[<span class="string">&#x27;living&#x27;</span>])</span><br><span class="line">generate_model(cfd, <span class="string">&#x27;living&#x27;</span>)</span><br></pre></td></tr></table></figure></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载停用词语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> stopwords</span><br><span class="line"><span class="comment"># 加载英语中的停用词</span></span><br><span class="line">stopwords.words(<span class="string">&#x27;English&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载单词列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> words</span><br><span class="line">words.words()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载人名表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载发音词典</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> cmudict</span><br><span class="line">cmudict.entries()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载比较词表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> swadesh</span><br><span class="line"><span class="comment"># 比较词表的标识码</span></span><br><span class="line">swadesh.fileids()</span><br><span class="line"><span class="comment"># 输出比较词汇中的`en`文本资料</span></span><br><span class="line">swadesh.words(<span class="string">&#x27;en&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载词汇工具</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> toolbox</span><br><span class="line"><span class="comment"># 输出罗托卡特语（Rotokas）的词典文本资料</span></span><br><span class="line">toolbox.entries(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载WordNet</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> wordnet <span class="keyword">as</span> wn</span><br><span class="line"><span class="comment"># 输出`motocar`的同义词</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;motorcar&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义中的同义词集</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemma_names</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义的定义</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).definition</span><br><span class="line"><span class="comment"># 输出同义词集的所有词条</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemmas</span><br><span class="line"><span class="comment"># 输出特定的词条</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出一个词条对应的同义词集</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).synset</span><br><span class="line"><span class="comment"># 输出一个词条的&quot;名字&quot;</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).name</span><br><span class="line"><span class="comment"># 输出所有包含词`car`的词条</span></span><br><span class="line">wn.lemmas(<span class="string">&#x27;car&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取cat词性为名词的意义</span></span><br><span class="line">motorcar = wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 获取motorcar的下位词列表</span></span><br><span class="line">types_of_motorcar = motorcar.hyponyms()</span><br><span class="line"><span class="comment"># 获取motorcar的上位词列表</span></span><br><span class="line">motorcar.hypernyms()</span><br><span class="line"><span class="comment"># 获取motorcar上位词的路径列表</span></span><br><span class="line">paths = motorcar.hypernym_paths()</span><br><span class="line"><span class="comment"># 得到motorcar的根上位（最一般的上位）同义词集</span></span><br><span class="line">motorcar.root_hypernyms()</span><br><span class="line"><span class="comment"># 打开图形化WordNet的浏览器，更加便捷探索关系</span></span><br><span class="line">nltk.app.wordnet()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更多词汇关系</span></span><br><span class="line"><span class="comment"># 整体到部分：想象WordNet网络关系是一棵树</span></span><br><span class="line"><span class="comment"># 得到树的一部分，如树干、树冠等</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).part_meronyms()</span><br><span class="line"><span class="comment"># 得到树的实质，即心材和边材</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).substance_meronyms()</span><br><span class="line"><span class="comment"># 得到许多树构成的集合，即森林</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).member_holonyms()</span><br><span class="line"><span class="comment"># 得到mint所有名词意义的列表</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;mint&#x27;</span>, wn.NOUN)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 想象走路的动作</span></span><br><span class="line"><span class="comment"># 找出走路的一部分动作：抬脚</span></span><br><span class="line">wn.synset(<span class="string">&#x27;walk.v.01&#x27;</span>).entailments()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 反义词</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;supply.n.02.supply&#x27;</span>).antonyms()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用dir()查看词汇关系和同义词集上的定义</span></span><br><span class="line"><span class="built_in">dir</span>(wn.synset(<span class="string">&#x27;harmony.n.02&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 语义相似度</span></span><br><span class="line"><span class="comment"># 初始化一个同义词集中的两个单词</span></span><br><span class="line">right = wn.synset(<span class="string">&#x27;right_whale.n.01&#x27;</span>)</span><br><span class="line">minke = wn.synset(<span class="string">&#x27;minke_whale.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 如果它们有共同的上位词（且层次结构较低），则它们的关系一定十分密切</span></span><br><span class="line">right.lowest_common_hypernyms(minke)</span><br><span class="line"><span class="comment"># 查找同义词集深度量化关系，输出为数值。越小表示离根节点越近</span></span><br><span class="line">wn.synset(<span class="string">&#x27;baleen_whale.n.01&#x27;</span>).min_depth()</span><br><span class="line"><span class="comment"># 两个词 基于上位词层次结构中相互连接的概念之间的最短路径在0-1范围的打分（二者没有关系就返回1）</span></span><br><span class="line">right.path_similarity(minke)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其他相似性度量方法，可以通过help(wn)获取。比如:VerbNet</span></span><br></pre></td></tr></table></figure>
-<p>概念补充：</p>
-<ul>
-<li><p>文本语料库的结构：</p>
+<p>方阵：行数列数相同的矩阵</p>
+<p>对称矩阵：<span
+class="math inline">\(A=A^T\)</span>的矩阵，矩阵等于它自己的转置矩阵</p>
+<p>反对称矩阵：<span class="math inline">\(A=-A^T\)</span></p>
+<p>埃尔米特矩阵（Hermitian matrix）：<span
+class="math inline">\(A=A^*\)</span>，矩阵等于它自己的共轭转置矩阵</p>
+<p>斜埃尔米特矩阵：反埃尔米特矩阵，<span class="math inline">\(A =
+−A^*\)</span></p>
+<p>正定矩阵：<span class="math inline">\(Q(x)=x^T A
+x\)</span>，任意非零向量<span
+class="math inline">\(x\)</span>，若满足<span
+class="math inline">\(Q(x)&gt;0\)</span>，则<span
+class="math inline">\(A\)</span>为正定矩阵</p>
+<p>半正定矩阵：同上，<span class="math inline">\(Q(x) \geq
+0\)</span>，则A为半正定矩阵</p>
+<p>半负定矩阵：同上，<span class="math inline">\(Q(x) \leq
+0\)</span>，则A为半负定矩阵</p>
+<p>不定矩阵：若A既不是半正定矩阵，也不是半负定矩阵，则A为不定矩阵</p>
+<p>上三角矩阵：矩阵对角线左下方系数全为0</p>
+<p>下三角矩阵：矩阵对角线右上方系数全为0</p>
+<p>三对角矩阵：非零系数在主对角线上，也在比主对角线低和高一行的对角线上。</p>
+<p>单位矩阵：<span class="math inline">\(I\)</span>,
+矩阵对角线元素为1，其余为0</p>
+<p>高斯矩阵：首先满足单位矩阵，其次某列系数非0</p>
+<p>逆矩阵：反矩阵，<span
+class="math inline">\(AB=I\)</span>，A是可逆的，B是A的逆矩阵</p>
+<p>伴随矩阵：<strong>A</strong>关于第<em>i</em>行第<em>j</em>列的<strong>余子式</strong>，是去掉<strong>A</strong>的第<em>i</em>行第<em>j</em>列之后得到的(<em>n</em>
+− 1)×(<em>n</em> − 1)矩阵的行列式。</p>
+<p>阶梯型矩阵：从上至下，<span
+class="math inline">\(A_{ii}\)</span>之前的元素为0</p>
+<p>酉矩阵：<span class="math inline">\(UU^*=U^*U=I\)</span>，<span
+class="math inline">\(U^*\)</span>为U矩阵的共轭转置，则U为酉矩阵</p>
+<p>对角矩阵：只有矩阵中的对角线存在值，其他地方为0</p>
+<p>空矩阵：行数或列数为零的矩阵</p>
+<p>分块矩阵：是指一个大矩阵分割成“矩阵的矩阵“</p>
+<p>正交矩阵：满足方阵，<span
+class="math inline">\(Q^T=Q^{-1}\)</span>，<span
+class="math inline">\(Q^TQ=QQ^T=I\)</span>，正交矩阵的行列式为1</p>
+<p>旋转矩阵：在乘以一个向量的时候改变向量的方向但不改变大小的效果并保持了手性的矩阵</p>
+<p>正规矩阵：<span class="math inline">\(A^{*}A=AA^*\)</span>，<span
+class="math inline">\(A^*\)</span>为<span
+class="math inline">\(A\)</span>的共轭矩阵</p>
+<p>共轭矩阵：<span class="math inline">\(A^*=(\bar{A})^T\)</span>，<span
+class="math inline">\(\bar{A}\)</span>表示矩阵的复共轭</p>
+<p>奇异矩阵：满足方阵，矩阵的行列式是否等于0，若等于0，称矩阵A为奇异矩阵</p>
+<p>非奇异矩阵：同上，若不等于0，称矩阵A为非奇异矩阵。</p>
+<p>稀疏矩阵：若数值为0的元素数目远远多于非0元素的数目，并且非0元素分布没有规律时</p>
+<p>稠密矩阵：与上相反，若非0元素数目占大多数时</p>
+<p>增广矩阵：扩增矩阵，在系数矩阵的右边添上一列，这一列是线性方程组的等号右边的值</p>
+<p>系数矩阵：将方程组的系数组成矩阵来计算方程的解</p>
+<p>相似矩阵：判断特征值、行列式、迹、秩是否相等，只作为判断矩阵是否相似的必要条件，而非充分条件。</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>数学</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Ubuntu20.04安装深度学习环境及配置个人环境小记</title>
+    <url>/hexo_blog/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</url>
+    <content><![CDATA[<p>Ubuntu20.04安装深度学习环境及配置个人环境小记</p>
+<span id="more"></span>
+<h3 id="目标配置环境">目标配置环境</h3>
+<p>系统为Ubuntu 20.04,
+显卡为3090，结合其他服务器的环境，准备配置如下环境</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nvidia-driver-460 <span class="comment"># 英伟达驱动，最低推荐版本为这个</span></span><br><span class="line">CUDA 11.2 (gcc 7) <span class="comment"># 30系显卡应该换为cuda11版本以上，20系显卡可以用cuda10.2版本</span></span><br><span class="line">cudnn 8.6.1</span><br><span class="line"></span><br><span class="line">pytorch 1.10</span><br><span class="line">torchvision 0.11.1</span><br><span class="line">tensorflow 2.5.0</span><br></pre></td></tr></table></figure>
+<h3 id="准备工作">准备工作</h3>
+<p>刚装的系统，啥也没有，所以按照个人习惯先安装一些软件。为方便起见，后文非特殊说明，均用root用户完成安装。可以直接复制下文至sh文件中，更改权限以直接运行。（请保证软件源可正常连接）</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一步: 安装一些必备的软件</span></span><br><span class="line">add-apt-repository ppa:graphics-drivers/ppa</span><br><span class="line">apt update -y &amp;&amp; apt upgrade -y</span><br><span class="line">apt install -y git autossh python3-pip vim tmux gpustat tree ranger build-essential gcc-7 g++-7 make curl htop ipython3 zhs</span><br><span class="line"><span class="comment"># gcc与g++降级</span></span><br><span class="line">update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100</span><br><span class="line">update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步：安装nvidia驱动</span></span><br><span class="line"><span class="comment"># 禁用nouveau</span></span><br><span class="line"><span class="built_in">chmod</span> 666 /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 增加相应配置, 若已追加则记得注释下面这一行命令</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;</span></span><br><span class="line"><span class="string">blacklist vga16fb</span></span><br><span class="line"><span class="string">blacklist nouveau</span></span><br><span class="line"><span class="string">blacklist rivafb</span></span><br><span class="line"><span class="string">blacklist rivatv</span></span><br><span class="line"><span class="string">blacklist nvidiafb</span></span><br><span class="line"><span class="string">&quot;</span></span><br><span class="line">&gt;&gt; /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 更新配置</span></span><br><span class="line">update-initramfs -u</span><br><span class="line">apt install nvidia-driver-460 -y <span class="comment"># 英伟达460驱动，按需更改</span></span><br></pre></td></tr></table></figure>
+<p>执行完成之后，重启服务器。使用<code>nvidia-smi</code>查看gpu是否正常，或使用安装好的<code>gpustat</code></p>
+<h3 id="cuda及cudnn安装">cuda及cudnn安装</h3>
+<h4 id="下载">下载</h4>
+<p>这里需要从官网下载安装包，cuda安装包可以直接通过命令下载。cudnn需要注册英伟达开发者账号，下载四个文件</p>
 <ul>
-<li><p>孤立的无特别组织的文本集合</p></li>
-<li><p>按文体等分类组织结构</p></li>
-<li><p>分类重叠，主题类别（路透社语料库）</p></li>
-<li><p>随时间变化语言语言用法的改变（就职演说语料库）</p></li>
-</ul></li>
-<li><p>NLTK中定义的基本语料库函数</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-4.png" /></p></li>
-<li><p>条件频率分布</p>
-<p><strong>条件频率分布是频率分布的集合，每个频率分布有一个不同的“条件”。这个条件通常是文本的类别。</strong></p>
-<p>当语料文本被分为几类（文体、主题、作者等）时，我们可以计算每个类别独立的频率分布，研究类别之间的系统性差异。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-5.png" /></p></li>
-<li><p>条件和事件</p>
-<p>对于NLP来说，文本出现的词汇就是事件。条件频率分布需要给每个事件关联一个条件，所以处理的是一个配对序列。每对形式是：（条件，事件）。举例就是，（文本类别，文本中的一个词汇）</p></li>
-<li><p>FreqDist()以一个简单的链表作为输入，ConditionalFreqDist()以一个配对链表作为输入。</p></li>
-<li><p>停用词：
-如<code>the</code>、<code>to</code>这种高频词汇。这种词通常没有什么词汇内容，但是它们又会让文本分析变得困难。所以在需要的情况下，我们就需要从文档中过滤掉他们。</p></li>
-<li><p>WordNet：面向语义的英语词典，类似与传统辞典，但具有更丰富的结构。每个节点对应一个<strong>同义词集</strong>，边表示上/下位词关系（上下级概念与从属概念的关系）</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-6.png" /></p></li>
-<li><p>synset：同义词集，意义想相同的词（或“词条”）的集合。如car.n.01是car的第一个名词的意义，就是同义词集。</p></li>
-<li><p>词条：同义词集和词的配对。</p></li>
+<li><p><code>cudnn-11.2-linux-x64-v8.1.0.77</code></p></li>
+<li><p><code>libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+<li><p><code>libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+<li><p><code>libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
 </ul>
-<p>下一篇：<a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda_11.2.2_460.32.03_linux.run &amp;&amp; <span class="built_in">chmod</span> 755 cuda_11.2.2_460.32.03_linux.run</span><br><span class="line"></span><br><span class="line">./cuda_11.2.2_460.32.03_linux.run</span><br></pre></td></tr></table></figure>
+<p>下载完直接运行，安装过程中需要把Drivers这项取消，因为自带了显卡驱动。</p>
+<h4 id="环境变量">环境变量</h4>
+<p>在/etc/bash.bashrc或/etc/profile前加入如下两行。如果profile不行就换bashrc。一般是编辑<code>vim ~/.bashrc</code>下，但为保证所有用户都生效，所以需要在<code>/etc</code>目录下编辑</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">export</span> PATH=/usr/local/cuda-11.2/bin<span class="variable">$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span></span><br><span class="line"><span class="built_in">export</span> LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64<span class="variable">$&#123;LD_LIBRARY_PATH:+:<span class="variable">$&#123;LD_LIBRARY_PATH&#125;</span>&#125;</span></span><br></pre></td></tr></table></figure>
+<p>至此，cuda安装完成</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 检测是否安装成功</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/cuda/samples/1_Utilities/deviceQuery</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./deviceQuery</span><br><span class="line"><span class="comment"># 最后一行输出，Result = PASS表示成功</span></span><br></pre></td></tr></table></figure>
+<h4 id="cudnn配置">cudnn配置</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -zxvf cudnn-11.2-linux-x64-v8.1.0.77.tgz </span><br><span class="line"><span class="built_in">cp</span> cuda/include/cudnn.h /usr/local/cuda/include </span><br><span class="line"><span class="built_in">cp</span> cuda/lib64/libcudnn* /usr/local/cuda/lib64  </span><br><span class="line"><span class="built_in">chmod</span> a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*  </span><br><span class="line"></span><br><span class="line">dpkg -i libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb      </span><br><span class="line">dpkg -i libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb  </span><br><span class="line">dpkg -i libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb </span><br><span class="line"></span><br><span class="line"><span class="comment"># cudnn检测，更高版本的cudnn需要把cudnn.h换成cudnn_version.h</span></span><br><span class="line"><span class="built_in">cat</span> /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2</span><br><span class="line"><span class="comment"># 利用程序检查，可能需要重启</span></span><br><span class="line"><span class="built_in">cd</span> /usr/src/cudnn_samples_v7/conv_sample</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./conv_sample</span><br><span class="line"><span class="comment"># 最后一行输出，Test passed！表示成功</span></span><br></pre></td></tr></table></figure>
+<p>通过<code>ldconfig</code>可检查动态链接库是否链接正确。若有输出，可以使用<code>ln -sf</code>重新链接。</p>
+<h3 id="python库安装">Python库安装</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pip3 install tensorflow-gpu==2.5.0 -i https://pypi.tuna.tsinghua.edu.cn/simple</span><br><span class="line">pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html</span><br><span class="line">pip3 install black tqdm flask sklearn</span><br></pre></td></tr></table></figure>
+<h4 id="检查是否安装成功">检查是否安装成功</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.cuda.is_available()) </span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.is_available()) </span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.__version__)</span><br><span class="line"><span class="built_in">print</span>(torch.version.cuda)</span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.version())</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br><span class="line">gpu_device_name = tf.test.gpu_device_name()</span><br><span class="line"><span class="built_in">print</span>(gpu_device_name)   </span><br><span class="line">tf.test.is_gpu_available() <span class="comment"># 输出True</span></span><br></pre></td></tr></table></figure>
+<p>若tf未检测到GPU，可以看输出过程中的哪些动态库文件加载失败，手动配置一下。</p>
+<p>至此，深度学习安装环境配置完成。</p>
+<h3 id="个人环境">个人环境</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建用户</span></span><br><span class="line">adduser xxx</span><br><span class="line"><span class="comment"># 输入配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 增加root权限</span></span><br><span class="line"><span class="built_in">chmod</span> u+w /etc/sudoers</span><br><span class="line">vim /etc/sudoers</span><br><span class="line">​```</span><br><span class="line">root    ALL=(ALL:ALL) ALL</span><br><span class="line"><span class="comment"># 增加一行</span></span><br><span class="line">xxx    ALL=(ALL:ALL) ALL</span><br><span class="line">​```</span><br><span class="line"><span class="built_in">chmod</span> u-w /etc/sudoers</span><br></pre></td></tr></table></figure>
+<h4 id="ssh文件配置">ssh文件配置</h4>
+<p>方便起见直接把其他服务器的<code>.ssh</code>文件夹下的<code>id_rsa</code>，<code>id_rsa.pub</code>和<code>authorized_keys</code>复制过来。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 复制完成之后，修改对应权限才能使用</span></span><br><span class="line"><span class="built_in">chmod</span> 755 ~/.ssh/  </span><br><span class="line"><span class="built_in">chmod</span> 600 ~/.ssh/id_rsa ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys</span><br><span class="line"><span class="built_in">chmod</span> 644 ~/.ssh/known_hosts</span><br></pre></td></tr></table></figure>
+<h4 id="oh-my-zsh">oh-my-zsh</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载并安装</span></span><br><span class="line">wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装插件</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-autosuggestions <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-autosuggestions</span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-syntax-highlighting.git <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-syntax-highlighting</span><br><span class="line"></span><br><span class="line"><span class="comment"># vim ~/.zshrc, 最后一行加入如下配置</span></span><br><span class="line">plugins=(</span><br><span class="line">        git</span><br><span class="line">        zsh-autosuggestions</span><br><span class="line">        zsh-syntax-highlighting</span><br><span class="line">)</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/oh-my-zsh.sh</span><br></pre></td></tr></table></figure>
+<h4 id="配置vim">配置vim</h4>
+<p>编辑<code>.vimrc</code></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">set</span> number</span><br><span class="line"></span><br><span class="line">syntax on</span><br><span class="line">colorscheme darkblue</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">syntax <span class="built_in">enable</span></span><br><span class="line">filetype plugin on</span><br><span class="line"><span class="built_in">let</span> g:go_disable_autoinstall = 0</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_functions = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_methods = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_structs = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_operators = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_build_constraints = 1</span><br><span class="line"><span class="string">&quot; &quot;</span> syntax on</span><br><span class="line"><span class="string">&quot;&quot;</span> filetype plugin indent on</span><br><span class="line"><span class="built_in">let</span> g:pymode_python = <span class="string">&#x27;python3&#x27;</span></span><br><span class="line"><span class="built_in">set</span> number</span><br><span class="line"><span class="built_in">set</span> cin</span><br><span class="line"><span class="built_in">set</span> sw=8</span><br><span class="line"><span class="built_in">set</span> ts=8</span><br><span class="line"><span class="built_in">set</span> sm</span><br><span class="line">colorscheme darkblue</span><br><span class="line"><span class="built_in">set</span> viminfo=<span class="string">&#x27;10,\&quot;100,:20,%,n~/.viminfo </span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">au BufReadPost * if line(&quot;&#x27;</span>\&quot;<span class="string">&quot;) &gt; 0|if line(&quot;</span><span class="string">&#x27;\&quot;&quot;) &lt;= line(&quot;$&quot;)|exe(&quot;norm &#x27;</span>\&quot;<span class="string">&quot;)|else|exe &quot;</span>norm $<span class="string">&quot;|endif|endif</span></span><br></pre></td></tr></table></figure>
+<h4 id="远程连接">远程连接</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">autossh -gfCNR 远程端口:localhost:22（默认本地ssh端口） 公网用户名@公网IP</span><br></pre></td></tr></table></figure>
+<p>需要提前配好免密登录，远程端口在公网服务器上需要开放防火墙。</p>
+<p>远程连接登录<code>ssh 服务器用户@公网IP -p 远程端口</code></p>
 ]]></content>
       <categories>
-        <category>NLP</category>
+        <category>Linux</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
+        <tag>CUDA</tag>
+        <tag>PyTorch</tag>
+        <tag>Tensorflow</tag>
+        <tag>Ubuntu</tag>
+        <tag>CUDNN</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅲ</title>
-    <url>/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
-<p>接 <a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
+    <title>crontab命令</title>
+    <url>/hexo_blog/2017/08/24/%E8%BF%90%E7%BB%B4/crontab%E5%91%BD%E4%BB%A4/</url>
+    <content><![CDATA[<p>学会使用crontab定时任务，提高工作效率。详解了crontab的基本格式和例子，以及相关目录介绍和日志文件的设置。</p>
 <span id="more"></span>
-<h2 id="第六章学习分类文本">第六章：学习分类文本</h2>
-<p>目标：</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看当前用户的定时任务，也可以使用crontab -uroot -l查看指定用户的定时任务</span></span><br><span class="line">crontab -l</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编辑crontab定时任务，貌似是使用vim的编辑命令</span></span><br><span class="line">crontab -e</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其实基本上知道这两个就可以处理定时任务了</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除当前用户的所有定时任务，不知道为什么会有这种设定（手贱按下过一次，恢复起来真的惨）</span></span><br><span class="line">crontab -r</span><br><span class="line"><span class="comment"># 如果按到了，就去日志文件里面找记录，尽可能恢复命令吧</span></span><br></pre></td></tr></table></figure>
+<p>下面讲讲编辑任务的格式</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 基本格式</span></span><br><span class="line"></span><br><span class="line">  *  *  *  *  * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 分 时 日 月 周 需要执行的命令</span></span><br></pre></td></tr></table></figure>
+<p>可视化测试在线网站，https://crontab.guru/</p>
+<p>几个栗子</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每分钟执行一次command命令，空格表示间隔</span></span><br><span class="line">* * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时执行一次，‘/’表示频率</span></span><br><span class="line">0 */1 * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时的10分和40分各执行一次，‘,’表示并列</span></span><br><span class="line">10,40 * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每个星期一的10-14点的10，40执行，‘-’表述区间</span></span><br><span class="line">10,40 10-14 * * 1 <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每天的8，13，20点执行，注意此时表示分的那个*要替换为0</span></span><br><span class="line">0 8,13,20 * * * <span class="built_in">command</span></span><br></pre></td></tr></table></figure>
+<p>日志文件一般存放在
+<code>/var/log/cron*</code>中，不过我的Ubuntu好像没有开启cron日志，可以在
+<code>/etc/rsyslog.d/50-default.conf</code>中进行设置（Centos在
+<code>/etc/rsyslog.conf</code>中设置）</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/rsyslog.d/50-default.conf</span><br><span class="line"><span class="comment"># 找到cron*, 去掉前面的#注释即可</span></span><br><span class="line"><span class="comment"># 重启rsyslog</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service rsyslog  restart</span><br></pre></td></tr></table></figure>
+<p>crontab相关目录介绍</p>
+<p><code>/etc/anacrontab</code>:
+这个文件存着系统级的任务。它主要用来运行每日的(daily)，每周的(weekly),每月的(monthly)的任务。一般不在此文件安装自己的任务</p>
+<p><code>/etc/cron.d/</code>: 此目录下存放的是系统级任务的任务文件。</p>
+<p><code>/var/spool/cron/</code>:
+此目录下存放各个用户的任务文件。各个用户的任务存放在以自已用户名为文件名的任务文件中。此文件中的指令行没有用户域。</p>
+<p><code>/etc/crontab</code>:
+crontab的主要配置文件，执行了每小时、每日、每周、每月的脚本文件</p>
+<p><code>/etc/cron.hourly</code>、<code>/etc/cron.daily</code>、<code>/etc/cron.weekly</code>、<code>/etc/cron.monthly</code>：这四个目录下分别对应着每隔一段时间需要执行的脚本任务，脚本使用shell命令编写</p>
+<p><a href="https://segmentfault.com/a/1190000002628040"></a></p>
+<p><a
+href="http://www.cnblogs.com/peida/archive/2013/01/08/2850483.html">每天一个linux命令（50）：crontab命令</a></p>
+<p><a href="https://segmentfault.com/a/1190000002628040">Linux
+下执行定时任务 crontab 命令详解</a></p>
+]]></content>
+      <categories>
+        <category>Software</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>crontab</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>端口转发工具笔记（端口转发）</title>
+    <url>/hexo_blog/2020/08/08/%E8%BF%90%E7%BB%B4/firewall%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</url>
+    <content><![CDATA[<p>如何通过防火墙本机端口转发来实现SSH跳板登录。继
+<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，根据新的需求衍生出新技巧</p>
+<span id="more"></span>
+<p>继 <a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，根据新的需求衍生出新技巧</p>
+<h3 id="需求">需求</h3>
 <ol type="1">
-<li>找语言数据中的分类特征</li>
-<li>构建模型</li>
-<li>从模型中学习知识</li>
+<li>继跳板机中转登录后，觉得需要中转登录还是太麻烦，所以希望能够跟普通ssh一样登录</li>
 </ol>
-<p>分类学习，一般分为有监督学习、无监督学习、半监督学习。</p>
-<p>分类的定义：将给定的输入选择正确（合适）的类标签。这里值得注意的是，输入之间是相互独立的，标签集是预先定义好的。延伸的任务，每个实例（输入）可以划分为多个标签；分类问题不一定预先定义标签集，类似K-Means聚类；序列分类中，一个输入链表可以作为一个整体分类。</p>
-<p>在这里，我们预先得到一些包含正确标签的语料（实例/输入），根据这些数据建立得到的模型进行分类，我们称为有监督分类。有监督分类中，在训练过程中，需要将每个一个输入值转换为特征集（特征提取器）。特征集对应着输出类标签。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，提取输入字符串中最后一个字母。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="comment"># 返回特征集</span></span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;last_letter&#x27;</span>: word[-<span class="number">1</span>]&#125;</span><br><span class="line"><span class="comment"># 导入姓名语料库, 获取男女姓名列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 随机打乱男女姓名列表</span></span><br><span class="line">random.shuffle(names)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 划分训练集和测试集</span></span><br><span class="line">featuresets = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 利用贝叶斯分类来训练模型</span></span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行预测 </span></span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Neo&#x27;</span>))</span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Trinity&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 准确率</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最有效的五个特征（字母）</span></span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string">             last_letter = &#x27;k&#x27;              male : female =     43.4 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;a&#x27;            female : male   =     32.9 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;f&#x27;              male : female =     16.1 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;p&#x27;              male : female =     12.6 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;v&#x27;              male : female =     10.6 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span>                 </span><br><span class="line"><span class="comment"># 输出的比率称为 似然比，可以用于比较不同特征-结果关系</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 处理大型语料库时，构建一个包含每一个实例的特征的单独的链表会使用大量的内存。这种情况下，可以使用下面的函数，使返回一个行为像一个链表而不会在内存存储所有特征集的对象</span></span><br><span class="line"><span class="keyword">from</span> nltk.classify <span class="keyword">import</span> apply_features</span><br><span class="line">train_set = apply_features(gender_features, names[<span class="number">500</span>:])</span><br><span class="line">test_set = apply_features(gender_features, names[:<span class="number">500</span>])</span><br></pre></td></tr></table></figure>
-<h3 id="选取特征">选取特征</h3>
-<p>过拟合：用于一个给定的学习算法的特征的数目是有限的——如果提供太多的特征，那么该算法将高度依赖训练数据的特性而一般化到新的例子的效果不会很好。</p>
-<p>过拟合当运作在小训练集上时尤其会有问题。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，过拟合提取特征。这里请对比gender_features2与上一版本的区别</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features2</span>(<span class="params">name</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&quot;firstletter&quot;</span>] = name[<span class="number">0</span>].lower()</span><br><span class="line">    features[<span class="string">&quot;lastletter&quot;</span>] = name[–<span class="number">1</span>].lower()</span><br><span class="line">    <span class="keyword">for</span> letter <span class="keyword">in</span> <span class="string">&#x27;abcdefghijklmnopqrstuvwxyz&#x27;</span>:</span><br><span class="line">        features[<span class="string">&quot;count(%s)&quot;</span> % letter] = name.lower().count(letter)</span><br><span class="line">        features[<span class="string">&quot;has(%s)&quot;</span> % letter] = (letter <span class="keyword">in</span> name.lower())</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">featuresets = [(gender_features2(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 对比上一版本的预测结果，基本可以发现精确度下降了</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<p>一旦初始特征集被选定，完善特征集的一个非常有成效的方法是<strong>错误分析</strong>。首先，我们选择一个
-开发集，包含用于创建模型的语料数据。然后将这种开发集分为 训练集和
-开发测试集。训练集用于训练模型，开发测试集用于进行错误分析，测试集用于系统的最终评估。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 划分训练集、开发集、开发测试集</span></span><br><span class="line">train_names = names[<span class="number">1500</span>:]</span><br><span class="line">devtest_names = names[<span class="number">500</span>:<span class="number">1500</span>]</span><br><span class="line">test_names = names[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 错误分析</span></span><br><span class="line">errors = []</span><br><span class="line"><span class="keyword">for</span> (name, tag) <span class="keyword">in</span> devtest_names:</span><br><span class="line">    guess = classifier.classify(gender_features(name))</span><br><span class="line">    <span class="keyword">if</span> guess != tag:</span><br><span class="line">        errors.append((tag, guess, name))</span><br><span class="line">      </span><br><span class="line"><span class="comment"># 输出错误分类列表中，猜错的类别及名字</span></span><br><span class="line"><span class="keyword">for</span> (tag, guess, name) <span class="keyword">in</span> <span class="built_in">sorted</span>(errors):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;correct=%-8s guess=%-8s name=%-30s&#x27;</span> % (tag, guess, name))</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 观察输出进行分析，得到一些结果。</span></span><br><span class="line"><span class="comment"># yn结尾的名字显示以女性为主，虽然n结尾的名字往往是男性</span></span><br><span class="line"><span class="comment"># 以ch结尾的名字通常是男性，虽然h结尾的名字倾向于是女性</span></span><br><span class="line"><span class="comment"># 根据结论，调整特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;suffix1&#x27;</span>: word[-<span class="number">1</span>:], <span class="string">&#x27;suffix2&#x27;</span>: word[-<span class="number">2</span>:]&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"><span class="comment"># 最终结果理论上会进行提高</span></span><br></pre></td></tr></table></figure>
-<p>不断地进行错误分析改善特征提取器，可以不断提高预测的准确度。每次错误分析应该选取不同的开发测试/训练进行分割，以检查新改进分类器可能产生的新的错误模式。</p>
-<h3 id="文档分类">文档分类</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> movie_reviews</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据电影评论分类（&#x27;neg&#x27;, &#x27;pos&#x27;）获取电影评论</span></span><br><span class="line">documents = [(<span class="built_in">list</span>(movie_reviews.words(fileid)), category) <span class="keyword">for</span> category <span class="keyword">in</span> movie_reviews.categories() <span class="keyword">for</span> fileid <span class="keyword">in</span> movie_reviews.fileids(category)]</span><br><span class="line"><span class="comment"># 随机打乱数据</span></span><br><span class="line">random.shuffle(documents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立文档分类的特征提取器</span></span><br><span class="line"><span class="comment"># 使用FreqDist会比一般的list更快</span></span><br><span class="line">all_words = nltk.FreqDist(w.lower() <span class="keyword">for</span> w <span class="keyword">in</span> movie_reviews.words())</span><br><span class="line">word_features = <span class="built_in">list</span>(all_words.keys())[:<span class="number">2000</span>]</span><br><span class="line"><span class="comment"># 每个词是否在一个给定的文档中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">document_features</span>(<span class="params">document</span>):</span><br><span class="line">    document_words = <span class="built_in">set</span>(document)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> word_features:</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word] = (word <span class="keyword">in</span> document_words)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练文档分类模型</span></span><br><span class="line">featuresets = [(document_features(d), c) <span class="keyword">for</span> (d,c) <span class="keyword">in</span> documents]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">100</span>:], featuresets[:<span class="number">100</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 检验模型</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>) </span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string"> contains(unimaginative) = True              neg : pos    =      7.7 : 1.0</span></span><br><span class="line"><span class="string">    contains(schumacher) = True              neg : pos    =      7.5 : 1.0</span></span><br><span class="line"><span class="string">        contains(suvari) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">          contains(mena) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">        contains(shoddy) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="comment"># 显然可以发现，`unimaginative`这个词是负面的概率是正面的7.7倍</span></span><br></pre></td></tr></table></figure>
-<h3 id="词性标注">词性标注</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为词选择词性标记，通过词的后缀来选择</span></span><br><span class="line">suffix_fdist = nltk.FreqDist()</span><br><span class="line"><span class="keyword">for</span> word <span class="keyword">in</span> brown.words():</span><br><span class="line">    word = word.lower()</span><br><span class="line">    suffix_fdist[word[-<span class="number">1</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">2</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">3</span>:]] += <span class="number">1</span></span><br><span class="line">common_suffixes = <span class="built_in">list</span>(suffix_fdist.keys())[:<span class="number">100</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 现在通过特征提取器，计算哪个后缀最有信息量（最具代表性)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">word</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> suffix <span class="keyword">in</span> common_suffixes:</span><br><span class="line">        features[<span class="string">&#x27;endswith(%s)&#x27;</span> % suffix] = word.lower().endswith(suffix)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">tagged_words = brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 利用特征提取器选取特征</span></span><br><span class="line">featuresets = [(pos_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> tagged_words]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line"><span class="comment"># 利用决策树方法来训练模型</span></span><br><span class="line">classifier = nltk.DecisionTreeClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.classify(pos_features(<span class="string">&#x27;cats&#x27;</span>))</span><br><span class="line"><span class="comment"># 将nltk运算过程，以伪代码方式输出。设定决策树深度为4</span></span><br><span class="line">classifier.pseudocode(depth=<span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据上下文语境，更新特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">pos_features(brown.sents()[<span class="number">0</span>], <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 检验新的词性标注器性能</span></span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">featuresets = []</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> tagged_sents:</span><br><span class="line">    untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">    <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">        featuresets.append((pos_features(untagged_sent, i), tag))</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<p>以上是在结合整个词集做的标注器，但是往往针对某些特殊情况，如形容词后面有很大概率上是名词。对于这种情况，我们的词性标注器目前还没有进行独立对待。</p>
-<h3 id="序列分类">序列分类</h3>
-<p>联合分类器：捕捉相关的分类任务之间的依赖关系，收集有关输入，选择适当的标签</p>
-<p>连续分类（贪婪序列分类）：为第一个输入找到最有可能的类标签，然后使用这个问题的答案帮助找到下一个输入的最佳的标签。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i, history</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]&#125;</span><br><span class="line">	<span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = history[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="keyword">class</span> <span class="title class_">ConsecutivePosTagger</span>(nltk.TaggerI):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, train_sents</span>):</span><br><span class="line">        train_set = []</span><br><span class="line">        <span class="keyword">for</span> tagged_sent <span class="keyword">in</span> train_sents:</span><br><span class="line">            untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">            history = []</span><br><span class="line">            <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">                featureset = pos_features(untagged_sent, i, history)</span><br><span class="line">                train_set.append( (featureset, tag) )</span><br><span class="line">                history.append(tag)</span><br><span class="line">        <span class="variable language_">self</span>.classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tag</span>(<span class="params">self, sentence</span>):</span><br><span class="line">        history = []</span><br><span class="line">        <span class="keyword">for</span> i, word <span class="keyword">in</span> <span class="built_in">enumerate</span>(sentence):</span><br><span class="line">            featureset = pos_features(sentence, i, history)</span><br><span class="line">            tag = <span class="variable language_">self</span>.classifier.classify(featureset)</span><br><span class="line">            history.append(tag)</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">zip</span>(sentence, history)</span><br><span class="line">  </span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(tagged_sents) * <span class="number">0.1</span>)</span><br><span class="line">train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]</span><br><span class="line">tagger = ConsecutivePosTagger(train_sents)</span><br><span class="line">tagger.evaluate(test_sents)</span><br></pre></td></tr></table></figure>
-<h3 id="其他序列模型">其他序列模型</h3>
-<p>以上模型有一个致命问题就是无法修复已经标注错误的词性。另一种方案<strong>隐马尔科夫模型</strong>可以为所有的可能序列进行打分，选择得分最高的序列。这种模型的缺点就是计算量相当大，可以采用动态规划来解决这种问题。基于这种模型，产生了<strong>最大熵马尔可夫模型</strong>和<strong>线性链条件随机场模型</strong>，二者为可能序列打分的算法不同。</p>
-<h3 id="句子分割">句子分割</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sents = nltk.corpus.treebank_raw.sents()</span><br><span class="line"><span class="comment"># 单独的句子标识符的合并链表</span></span><br><span class="line">tokens = []</span><br><span class="line"><span class="comment"># 所有句子边界标识索引</span></span><br><span class="line">boundaries = <span class="built_in">set</span>()</span><br><span class="line">offset = <span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> sent <span class="keyword">in</span> sents:</span><br><span class="line">    tokens.extend(sent)</span><br><span class="line">    offset += <span class="built_in">len</span>(sent)</span><br><span class="line">    boundaries.add(offset-<span class="number">1</span>)</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">punct_features</span>(<span class="params">tokens, i</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;next-word-capitalized&#x27;</span>: tokens[i+<span class="number">1</span>][<span class="number">0</span>].isupper(),</span><br><span class="line">            <span class="string">&#x27;prevword&#x27;</span>: tokens[i-<span class="number">1</span>].lower(),</span><br><span class="line">            <span class="string">&#x27;punct&#x27;</span>: tokens[i],</span><br><span class="line">            <span class="string">&#x27;prev-word-is-one-char&#x27;</span>: <span class="built_in">len</span>(tokens[i-<span class="number">1</span>]) == <span class="number">1</span>&#125;</span><br><span class="line"></span><br><span class="line">featuresets = [(punct_features(tokens, i), (i <span class="keyword">in</span> boundaries))</span><br><span class="line">				<span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="built_in">len</span>(tokens)-<span class="number">1</span>)</span><br><span class="line">				<span class="keyword">if</span> tokens[i] <span class="keyword">in</span> <span class="string">&#x27;.?!&#x27;</span>]</span><br><span class="line"><span class="comment"># 进行训练，检验模型准确率</span></span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<h3 id="识别对话行为类型">识别对话行为类型</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取即时消息会话语料库</span></span><br><span class="line">posts = nltk.corpus.nps_chat.xml_posts()[:<span class="number">10000</span>]</span><br><span class="line"><span class="comment"># 特征提取器，每个帖子包含什么词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">dialogue_act_features</span>(<span class="params">post</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> nltk.word_tokenize(post):</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word.lower()] = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="comment"># 构建模型，判断是否为对话行为</span></span><br><span class="line">featuresets = [(dialogue_act_features(post.text), post.get(<span class="string">&#x27;class&#x27;</span>)) <span class="keyword">for</span> post <span class="keyword">in</span> posts]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<h3 id="识别文字蕴含">识别文字蕴含</h3>
-<p>识别文字蕴含（Recognizing textual entailment，RTE）是判断文本 T
-的一个给定片段是否蕴含着另一个叫做“假设”的文本。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">rte_features</span>(<span class="params">rtepair</span>):</span><br><span class="line">    <span class="comment">#  RTEFeatureExtractor 类建立了一个除去一些停用词后在文本和假设中都有的词汇包，然后计算重叠和差异</span></span><br><span class="line">    extractor = nltk.RTEFeatureExtractor(rtepair)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&#x27;word_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;word_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 原文中rte识别率只是刚刚超过了58%，所以不多做介绍</span></span><br><span class="line">nltk.classify.rte_classify </span><br></pre></td></tr></table></figure>
-<h3 id="涉及到的定义">涉及到的定义</h3>
-<p>文中介绍一些关于评估模型、决策树、朴素贝叶斯分类器的知识。由于笔者已掌握这些基础模型知识所以在这里略去，以下介绍一些基本的定义。</p>
-<p>测试集：选择适当的比例，防止过拟合</p>
-<p>准确度：测量测试集上分类器正确标注的输入的比例。</p>
-<p>精确度（Precision），表示我们发现的项目中有多少是相关的，TP/(TP+
-FP)。</p>
-<p>召回率（Recall），表示相关的项目中我们发现了多少，TP/(TP+ FN)。</p>
-<p>F- 度量值（F-Measure）（或
-F-得分，F-Score），组合精确度和召回率为一个单独的得分，被定义为精确度和召回率的调和平均数(2
-× Precision × Recall)/(Precision+Recall)。</p>
-<p>混淆矩阵：其中每个 cells[i,j]表示正确的标签 i 被预测为标签 j 的次数
-。因此，对角线项目（即
-cells[i,i]）表示正确预测的标签，非对角线项目表示错误。nltk中可以使用
-<code>nltk.ConfusionMatrix</code>函数</p>
-<p>交叉验证：在不同的测试集上执行多个评估，然后组合这些评估的得分</p>
-<p>信息增益：用给定的特征分割输入值，衡量数据变得更有序的程度</p>
-<p>朴素贝叶斯假设（独立性假设）：每个输入值是通过首先为那个输入值选择一个类标签，然后产生每个特征的方式产生的
-，每个特征与其他特征完全独立。</p>
-<p>零计数：如果训练集中有特征从来没有和给定标签一起出现，导致给定标签的标签可能性为
-0。</p>
-<p>平滑：给定一个初值，解决零计数问题。<code>nltk.probability</code>提供了多种平滑技术</p>
-<p>Heldout估计使用一个heldout
-语料库计算特征频率与特征概率之间的关系。</p>
-<p>非二元特征：如果标签是1,2,4,5这种多元标签，可以转换0&lt;x&lt;3,
-3&lt;x&lt;6这种二元标签</p>
-<p>双重计数：在训练过程中特征的贡献被分开计算，但当使用分类器为新输入选择标签时，这些特征的贡献被组合。解决方案就是为每一个特征贡献设定一个权重。</p>
-<p>最大熵分类器：与朴素贝叶斯类似，使用搜索技术找出一组将最大限度地提高分类器性能的参数。避免使用广义
-迭代缩放（Generalized Iterative Scaling，GIS）或改进的迭代缩放（Improved
-Iterative Scaling，IIS），这两者都比共轭梯度（Conjugate Gradient，CG
-）和 BFGS 优化方法慢很多</p>
-<p>联合特征：每个接收它自己的参数的标签和特征的组合。联合特征是有标签的的值的属性，而（简单）特征是未加标签的值的属性。</p>
-<p>一般情况下，
-最大熵原理是说在与我们所知道的一致的的分布中，我们会选择熵最高（分布最均匀）的。</p>
-<p>对于每个联合特征，最大熵模型计算该特征的“经验频率”——即它出现在训练集中的频率。然后，它搜索熵最大的分布，同时也预测每个联合特征正确的频率。</p>
-<p>一般情况下，生成式模型确实比条件式模型强大。但是前者所需的参数会大于后者。</p>
+<h3 id="工具">工具</h3>
+<ol type="1">
+<li>防火墙配置（iptables/ufw/firewall）</li>
+<li>ssh</li>
+</ol>
+<h3 id="先验知识">先验知识</h3>
+<ol type="1">
+<li><a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a></li>
+<li>防火墙配置
+<ol type="1">
+<li>iptables（Centos7之前自带）</li>
+<li>firewall（Centos7之后自带）</li>
+<li>ufw（Ubuntu自带）</li>
+<li>netsh（Windows自带）</li>
+</ol></li>
+</ol>
+<h3 id="实验">实验</h3>
+<ol type="1">
+<li>假设内网机器A，内网机器C，公网机器B。A通过将22端口转发至B的3000端口来进行跳板登录。所以C需要执行如下命令来登录A</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># C登录B</span></span><br><span class="line">ssh userB@ip_B</span><br><span class="line"><span class="comment"># B登录A</span></span><br><span class="line">ssh userA@127.0.0.1 -p 3000</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li>通过防火墙本机端口转发，将本机的3000端口转发至3000端口</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 两者的前提需要在/etc/sysctl.conf中，令net.ipv4.ip_forward = 1</span></span><br><span class="line"><span class="comment"># 使配置生效</span></span><br><span class="line">sysctl -p</span><br><span class="line"></span><br><span class="line"><span class="comment"># iptables</span></span><br><span class="line">iptables -t nat -A PREROUTING -p tcp -i eth0 -d 127.0.0.1 --dport 3000 -j DNAT --to 127.0.0.1:3000</span><br><span class="line"></span><br><span class="line"><span class="comment"># firewall</span></span><br><span class="line"><span class="comment"># 添加端口</span></span><br><span class="line">firewall-cmd --add-port=3000/tcp --permanent</span><br><span class="line"><span class="comment"># 端口转发</span></span><br><span class="line">firewall-cmd --permanent --add-forward-port=port=3000:proto=tcp:toport=3000 --permanent</span><br><span class="line"></span><br><span class="line"><span class="comment"># --permanent 永久生效</span></span><br><span class="line"><span class="comment"># firewall添加后，需要重启</span></span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment"># netsh</span></span><br><span class="line">netsh interface portproxy add v4tov4 listenaddress=127.0.0.1 listenport=2121 connectaddress=ip_B connectport=222</span><br></pre></td></tr></table></figure>
+<ol start="3" type="1">
+<li>以上，即可通过如下命令直接登录A</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ssh userA@ip_B -p 3000</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>NLP</category>
+        <category>Software</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
+        <tag>Linux</tag>
+        <tag>端口转发</tag>
+        <tag>ssh</tag>
+        <tag>iptables</tag>
+        <tag>firewall</tag>
+        <tag>ufw</tag>
+        <tag>netsh</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅱ</title>
-    <url>/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
-<p>接 <a href="/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/" title="NLTK阅读笔记Ⅰ">NLTK阅读笔记Ⅰ</a></p>
+    <title>ssh笔记（端口转发）</title>
+    <url>/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/</url>
+    <content><![CDATA[<p>使用SSH实现本地和远程主机之间的端口转发。本文介绍了SSH端口转发的基础操作，参数详解和用途介绍，并提供了一键命令和注意事项。SSH端口转发可以用于内外网穿透，实现服务器代理，动态转发端口，实现远端操作等功能。</p>
 <span id="more"></span>
-<h2 id="第三章加工原料文本">第三章：加工原料文本</h2>
-<p><a href="http://www.gutenberg.org/catalog/">古腾堡项目</a></p>
-<p>可以从这个项目中获取感兴趣的文本，之后进行分析。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 将原始文本字符串转换为list，以\r\n标点符号等为分隔符进行切割</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line"><span class="comment"># 创建NLTK文本</span></span><br><span class="line">text = nltk.Text(tokens)</span><br></pre></td></tr></table></figure>
-<p>处理文本部分：</p>
-<p><code>requests</code>： 数据获取</p>
-<p><code>BeautifulSoup</code>、<code>re</code>、<code>xpath</code>：
-文本处理</p>
-<p><code>feedparser</code>：处理RSS订阅</p>
-<p><code>os</code>：处理本地文件</p>
-<p>字符串基本操作在这里请见下图。注：字符串是不可改变的。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-2.png" /></p>
-<p>关于编码这个大坑，在Python3中做的已经比Python2好太多，在这里也不做介绍了。介绍一些关于编码处理的库。</p>
-<p><code>codecs</code>、<code>unicodedata</code></p>
-<p>正则表达式部分，推荐自主学习。难度系数不算高。搜索关键词，Python,
-正则表达式,
-re即可。值得一提的是，<code>re</code>是目前我了解处理文本速度最快最优雅的一种方式。</p>
-<p>英文文本是高度冗余的，忽略掉词内部的元音仍然可以很容易的阅读，有些时候这很明显。例如：declaration
-变成 dclrtn，inalienable 变成 inlnble，保留所有词首或词尾的元音序列
-。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">regexp = <span class="string">r&#x27;^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]&#x27;</span></span><br><span class="line"><span class="comment"># 压缩元音字符串</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compress</span>(<span class="params">word</span>):</span><br><span class="line">	pieces = re.findall(regexp, word)</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;&#x27;</span>.join(pieces)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取语料库数据</span></span><br><span class="line">english_udhr = nltk.corpus.udhr.words(<span class="string">&#x27;English-Latin1&#x27;</span>)</span><br><span class="line"><span class="comment"># 进行压缩</span></span><br><span class="line">nltk.tokenwrap(compress(w) <span class="keyword">for</span> w <span class="keyword">in</span> english_udhr[:<span class="number">75</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则结合条件频率，提取辅音-元音序列</span></span><br><span class="line">rotokas_words = nltk.corpus.toolbox.words(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br><span class="line">cvs = [cv <span class="keyword">for</span> w <span class="keyword">in</span> rotokas_words <span class="keyword">for</span> cv <span class="keyword">in</span> re.findall(<span class="string">r&#x27;[ptksvr][aeiou]&#x27;</span>, w)]</span><br><span class="line">cfd = nltk.ConditionalFreqDist(cvs)</span><br><span class="line">cfd.tabulate()</span><br><span class="line"></span><br><span class="line">    a   e   i   o   u</span><br><span class="line">k <span class="number">418</span> <span class="number">148</span>  <span class="number">94</span> <span class="number">420</span> <span class="number">173</span></span><br><span class="line">p  <span class="number">83</span>  <span class="number">31</span> <span class="number">105</span>  <span class="number">34</span>  <span class="number">51</span></span><br><span class="line">r <span class="number">187</span>  <span class="number">63</span>  <span class="number">84</span>  <span class="number">89</span>  <span class="number">79</span></span><br><span class="line">s   <span class="number">0</span>   <span class="number">0</span> <span class="number">100</span>   <span class="number">2</span>   <span class="number">1</span></span><br><span class="line">t  <span class="number">47</span>   <span class="number">8</span>   <span class="number">0</span> <span class="number">148</span>  <span class="number">37</span></span><br><span class="line">v  <span class="number">93</span>  <span class="number">27</span> <span class="number">105</span>  <span class="number">48</span>  <span class="number">49</span></span><br></pre></td></tr></table></figure>
-<h3 id="词干提取器">词干提取器</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查找词干，laptop与laptops其实只是单复数的区别。词干是相同的</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">stem</span>(<span class="params">word</span>):</span><br><span class="line">    regexp = <span class="string">r&#x27;^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$&#x27;</span></span><br><span class="line">    stem, suffix = re.findall(regexp, word)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> stem</span><br><span class="line"></span><br><span class="line"><span class="comment"># nltk自带的词干提取器</span></span><br><span class="line"><span class="comment"># 索引一些文本和使搜索支持不同词汇形式的话，使用Porter词干提取器</span></span><br><span class="line">porter = nltk.PorterStemmer()</span><br><span class="line">lancaster = nltk.LancasterStemmer()</span><br><span class="line"><span class="comment"># 编译一些文本的词汇或想要一个有效词条（或中心词）列表，使用WordNet词形归并器</span></span><br><span class="line">wnl = nltk.WordNetLemmatizer()</span><br></pre></td></tr></table></figure>
-<h3 id="分词">分词：</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用正则进行分词</span></span><br><span class="line">re.findall(<span class="string">r&quot;\w+(?:[-&#x27;]\w+)|&#x27;|[-.(]+|\S\w&quot;</span>, raw)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用nltk自带的函数进行分词，效率更高且不需要特殊处理符号</span></span><br><span class="line">nltk.regexp_tokenize()</span><br><span class="line"></span><br><span class="line">pattern = <span class="string">r&#x27;&#x27;&#x27;(?x) # 去除嵌入的空白字符和注释</span></span><br><span class="line"><span class="string">			([A-Z]\.)+ # 设定允许的标识符（单词）</span></span><br><span class="line"><span class="string">			| \w+(-\w+)* # 可选的内部连字符</span></span><br><span class="line"><span class="string">			| \$?\d+(\.\d+)?%? # 货币与百分比</span></span><br><span class="line"><span class="string">			| \.\.\. # 省略符</span></span><br><span class="line"><span class="string">			| [][.,;&quot;&#x27;?():-_`] # 额外的标志</span></span><br><span class="line"><span class="string">		&#x27;&#x27;&#x27;</span></span><br><span class="line">nltk.regexp_tokenize(text, pattern)</span><br></pre></td></tr></table></figure>
-<p>已经分词好的数据举例：</p>
-<p>《华尔街日报》原始文本（<code>nltk.corpus.treebank_raw.raw()</code>）和分好词的版本（<code>nltk.corpus.treebank.words()</code>）</p>
-<p>分词的最后一个问题是缩写的存在，如“didn't”。如果我们想分析一个句子的意思，将这种形式规范化为两个独立的形式：“did”和“n't”(不是
-not)可能更加有用。我们可以通过查表来做这项工作。</p>
-<p>链表与字符串部分不做介绍，是Python字符串的常规操作</p>
-<p>补充部分：</p>
-<ul>
-<li><p>建立词汇表流程</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-1.png" /></p></li>
-<li><p>编码点：每个字符分配一个编号</p></li>
-<li><p>解码：通过一些机制来将文本翻译成 Unicode的过程</p></li>
-<li><p>编码：将Unicode转化为其它编码的过程</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-3.png" /></p></li>
-<li><p>从 Unicode的角度来看，字符是可以实现一个或多个
-字形的抽象的实体。只有字形可以出现在屏幕上或被打印在纸上。一个字体是一个字符到字形映射。</p></li>
-<li><p>规范化文本：如将所有的文本小写保存，使用<code>lower()</code>这个函数</p></li>
-<li><p>​断句：在将文本分词之前，我们需要将它分割成句子</p></li>
-<li><p>分词（重点内容）：</p>
-<ol type="1">
-<li><p>找到一种方法来分开文本内容与分词标志。给每个字符标注一个布尔值来指示这个字符后面是否有一个分词标志</p></li>
-<li><p>找到将文本字符串正确分割成词汇的字位串。根据一个字典，来根据字典中词的序列来重构源文本。定义一个目标函数，通过字典大小与重构源文本所需的信息量尽力优化它的值。</p></li>
-<li><p>找到最大化目标函数值的0和1的模式。使用<strong>模拟退火算法</strong>的非确定性搜索。</p></li>
-</ol></li>
-</ul>
-<h2 id="第四章编写结构化程序">第四章：编写结构化程序</h2>
-<p>大量关于Python代码，不做介绍。此部分适宜阅读人群，使用Python时间不超过一年（不够熟练）</p>
-<p>文档说明模块：<code>docstring</code></p>
-<p>调试技术：<code>pdb</code></p>
-<p>绘图：<code>matplotlib</code></p>
-<p>图的绘制：<code>networkx</code></p>
-<p>数据统计表：<code>csv</code>（这里个人推荐有使用<code>pandas</code>，虽然它很”重“）</p>
-<p>数值运算：<code>numpy</code></p>
-<p>算法设计：</p>
-<ol type="1">
-<li>递归</li>
-<li>权衡空间与时间</li>
-<li>动态规划</li>
-</ol>
-<h2 id="第五章分类和标注词汇">第五章：分类和标注词汇</h2>
-<p>NLP基本技术：</p>
-<ul>
-<li>序列标注</li>
-<li>N-gram 模型</li>
-<li>回退和评估</li>
-</ul>
-<p>将词汇按它们的<strong>词性</strong>（parts-of-speech，POS）分类以及相应的标注它们的过程被称为<strong>词性标注</strong>（part-of-speech
-tagging, POS
-tagging）或干脆简称<strong>标注</strong>。词性也称为<strong>词类</strong>或<strong>词汇范畴</strong>。用于特定任务的标记的集合被称为一个<strong>标记集</strong>。</p>
-<h3 id="词性标注器">词性标注器</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line">text = nltk.word_tokenize(<span class="string">&quot;And now for something completely different&quot;</span>)</span><br><span class="line"><span class="comment"># 标注出text中每个词的词性</span></span><br><span class="line">nltk.pos_tag(text)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;And&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;now&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;for&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;something&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;completely&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;different&#x27;, &#x27;JJ&#x27;)]</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">text = nltk.Text(word.lower() <span class="keyword">for</span> word <span class="keyword">in</span> nltk.corpus.brown.words())</span><br><span class="line"><span class="comment"># 为一个词找出所有它的上下文</span></span><br><span class="line">text.similar(<span class="string">&#x27;woman&#x27;</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">man time day year car moment world house family child country boy</span></span><br><span class="line"><span class="string">state job place way war girl work word</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="标注语料库">标注语料库</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建一个有标识的字符串</span></span><br><span class="line">tagged_token = nltk.tag.str2tuple(<span class="string">&#x27;fly/NN&#x27;</span>)</span><br><span class="line">tagged_token</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">tagged_token</span></span><br><span class="line"><span class="string">(&#x27;fly&#x27;, &#x27;NN&#x27;)</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">tagged_token[<span class="number">0</span>]</span><br><span class="line">tagged_token[<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 批量标注</span></span><br><span class="line">sent = <span class="string">&#x27;&#x27;&#x27;The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/INother/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CCFulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPsaid/VBD ``/`` ARE/BER well/QL operated/VBN and/CC follow/VB generally/Raccepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJTinterest/NN of/IN both/ABX governments/NNS &#x27;&#x27;/&#x27;&#x27; ./.&#x27;&#x27;&#x27;</span></span><br><span class="line">[nltk.tag.str2tuple(t) <span class="keyword">for</span> t <span class="keyword">in</span> sent.split()]</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;The&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;grand&#x27;, &#x27;JJ&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;jury&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;commented&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;on&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;a&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;number&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of/INother&#x27;, &#x27;AP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;topics&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;,&#x27;, &#x27;,&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;AMONG&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;them&#x27;, &#x27;PPO&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;Atlanta&#x27;, &#x27;NP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and/CCFulton&#x27;, &#x27;NP-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;County&#x27;, &#x27;NN-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;purchasing&#x27;, &#x27;VBG&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;departments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;it/PPsaid&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;``&#x27;, &#x27;``&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;ARE&#x27;, &#x27;BER&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;well&#x27;, &#x27;QL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;operated&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;follow&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;generally/Raccepted&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;practices&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;inure&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;to&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;best/JJTinterest&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;both&#x27;, &#x27;ABX&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;governments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&quot;&#x27;&#x27;&quot;, &quot;&#x27;&#x27;&quot;),</span></span><br><span class="line"><span class="string"> (&#x27;.&#x27;, &#x27;.&#x27;)]</span></span><br><span class="line"><span class="string"> &#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="获取已标注的语料库">获取已标注的语料库</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 获取标注好的语料库</span></span><br><span class="line">nltk.corpus.brown.tagged_words()</span><br><span class="line"><span class="comment">#</span></span><br><span class="line">nltk.corpus.brown.tagged_words(tagset=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">nltk.corpus.nps_chat.tagged_words()</span><br><span class="line">nltk.corpus.conll2000.tagged_words()</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 未简化的标记</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 最频繁的名词标记</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">findtags</span>(<span class="params">tag_prefix, tagged_text</span>):</span><br><span class="line">	cfd = nltk.ConditionalFreqDist((tag, word) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> tagged_text <span class="keyword">if</span> tag.startswith(tag_prefix))</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">dict</span>((tag, cfd[tag].keys()[:<span class="number">5</span>]) <span class="keyword">for</span> tag <span class="keyword">in</span> cfd.conditions())</span><br><span class="line"></span><br><span class="line">tagdict = findtags(<span class="string">&#x27;NN&#x27;</span>, nltk.corpus.brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line"><span class="keyword">for</span> tag <span class="keyword">in</span> <span class="built_in">sorted</span>(tagdict):</span><br><span class="line">	<span class="built_in">print</span> tag, tagdict[tag]</span><br><span class="line"></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">$：名词所有格,</span></span><br><span class="line"><span class="string">S：复数名词（因为复数名词通常以 s结尾），</span></span><br><span class="line"><span class="string">P：专有名词</span></span><br><span class="line"><span class="string">-NC：引用</span></span><br><span class="line"><span class="string">-HL：标题中的词</span></span><br><span class="line"><span class="string">-TL：标题（布朗标记的特征）</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 探索已标注的语料库</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用POS标记寻找三词短语</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">process</span>(<span class="params">sentence</span>):</span><br><span class="line">	<span class="keyword">for</span> (w1,t1), (w2,t2), (w3,t3) <span class="keyword">in</span> nltk.trigrams(sentence):</span><br><span class="line">         <span class="comment"># 动词+TO+动词</span></span><br><span class="line">		<span class="keyword">if</span> (t1.startswith(<span class="string">&#x27;V&#x27;</span>) <span class="keyword">and</span> t2 == <span class="string">&#x27;TO&#x27;</span> <span class="keyword">and</span> t3.startswith(<span class="string">&#x27;V&#x27;</span>)):</span><br><span class="line">			<span class="built_in">print</span>(w1, w2, w3)</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> brown.tagged_sents():</span><br><span class="line">	process(tagged_sent)</span><br></pre></td></tr></table></figure>
-<p>Python字典部分不做介绍。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用默认字典可以防止使用未定义的key报错</span></span><br><span class="line"><span class="comment"># nltk的默认字典,需要提前定义类型（int、float、str、list、dict、tuple）。</span></span><br><span class="line">frequency = nltk.defaultdict(<span class="built_in">int</span>)</span><br><span class="line"><span class="comment"># 创建具有默认值的字典</span></span><br><span class="line">pos = nltk.defaultdict(<span class="keyword">lambda</span>: <span class="string">&#x27;N&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<h3 id="自动标注">自动标注</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 默认标注器</span></span><br><span class="line">tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)]</span><br><span class="line">nltk.FreqDist(tags).<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 将所有词都标注为NN的标注器</span></span><br><span class="line">raw = <span class="string">&#x27;I do not like green eggs and ham, I do not like them Sam I am!&#x27;</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line">default_tagger = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">default_tagger.tag(tokens)</span><br><span class="line"><span class="comment"># 评判标注正确性</span></span><br><span class="line">default_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则标注</span></span><br><span class="line">patterns = [</span><br><span class="line">	(<span class="string">r&#x27;.*ing$&#x27;</span>, <span class="string">&#x27;VBG&#x27;</span>), <span class="comment"># gerunds</span></span><br><span class="line">	(<span class="string">r&#x27;.*ed$&#x27;</span>, <span class="string">&#x27;VBD&#x27;</span>), <span class="comment"># simple past</span></span><br><span class="line">	(<span class="string">r&#x27;.*es$&#x27;</span>, <span class="string">&#x27;VBZ&#x27;</span>), <span class="comment"># 3rd singular present</span></span><br><span class="line">	(<span class="string">r&#x27;.*ould$&#x27;</span>, <span class="string">&#x27;MD&#x27;</span>), <span class="comment"># modals</span></span><br><span class="line">	(<span class="string">r&#x27;.*\&#x27;s$&#x27;</span>, <span class="string">&#x27;NN$&#x27;</span>), <span class="comment"># possessive nouns</span></span><br><span class="line">	(<span class="string">r&#x27;.*s$&#x27;</span>, <span class="string">&#x27;NNS&#x27;</span>), <span class="comment"># plural nouns</span></span><br><span class="line">	(<span class="string">r&#x27;^-?[0-9]+(.[0-9]+)?$&#x27;</span>, <span class="string">&#x27;CD&#x27;</span>), <span class="comment"># cardinal numbers</span></span><br><span class="line">	(<span class="string">r&#x27;.*&#x27;</span>, <span class="string">&#x27;NN&#x27;</span>) <span class="comment"># nouns (default)</span></span><br><span class="line">]</span><br><span class="line">regexp_tagger = nltk.RegexpTagger(patterns)</span><br><span class="line">regexp_tagger.tag(brown_sents[<span class="number">3</span>])</span><br><span class="line">regexp_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查询标注器</span></span><br><span class="line"><span class="comment"># 找出100个最频繁的词</span></span><br><span class="line">fd = nltk.FreqDist(brown.words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">most_freq_words = fd.keys()[:<span class="number">100</span>]</span><br><span class="line">likely_tags = <span class="built_in">dict</span>((word, cfd[word].<span class="built_in">max</span>()) <span class="keyword">for</span> word <span class="keyword">in</span> most_freq_words)</span><br><span class="line"><span class="comment"># 使用上面这个信息作为&quot;查找标注器&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags)</span><br><span class="line">baseline_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用信息作为标注器，如果没有找到就使用默认标注器。这个过程称为&quot;回退&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags, backoff=nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>))</span><br></pre></td></tr></table></figure>
-<h3 id="评估">评估</h3>
-<p>使用<strong>黄金标准</strong>测试数据。一个已经手动标注并作为自动系统评估标准而被接收的语料库。</p>
-<p>如果标注器对词的标记与黄金标准标记相同，那么标注器就被认为是正确的。当然这只是相对于黄金标准这个测试数据而言。</p>
-<p>关于开发一个已标注的语料库，这是一个庞大的任务，其中涉及到了许多方面。</p>
-<p>可以通过<code>nltk.app.concordance()</code>来可视化查找某个语料库中某个单词的词性</p>
-<table>
-<thead>
-<tr>
-<th>标记</th>
-<th>含义</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>ADJ</td>
-<td>形容词</td>
-</tr>
-<tr>
-<td>ADV</td>
-<td>动词</td>
-</tr>
-<tr>
-<td>CNJ</td>
-<td>连词</td>
-</tr>
-<tr>
-<td>CC</td>
-<td>并列连词</td>
-</tr>
-<tr>
-<td>DET</td>
-<td>限定词</td>
-</tr>
-<tr>
-<td>EX</td>
-<td>存在量词</td>
-</tr>
-<tr>
-<td>FW</td>
-<td>外来词</td>
-</tr>
-<tr>
-<td>MOD</td>
-<td>情态动词</td>
-</tr>
-<tr>
-<td>RB</td>
-<td>副词</td>
-</tr>
-<tr>
-<td>IN</td>
-<td>介词</td>
-</tr>
-<tr>
-<td>N</td>
-<td>名词</td>
-</tr>
-<tr>
-<td>NP</td>
-<td>专有名词</td>
-</tr>
-<tr>
-<td>NUM</td>
-<td>数词</td>
-</tr>
-<tr>
-<td>PRO</td>
-<td>代词</td>
-</tr>
-<tr>
-<td>P</td>
-<td>介词</td>
-</tr>
-<tr>
-<td>TO</td>
-<td>词to</td>
-</tr>
-<tr>
-<td>UH</td>
-<td>感叹词</td>
-</tr>
-<tr>
-<td>V</td>
-<td>动词</td>
-</tr>
-<tr>
-<td>VD</td>
-<td>过去式</td>
-</tr>
-<tr>
-<td>VG</td>
-<td>现在分词</td>
-</tr>
-<tr>
-<td>VN</td>
-<td>过去分词</td>
-</tr>
-<tr>
-<td>WH</td>
-<td>Wh限定词</td>
-</tr>
-<tr>
-<td>NN</td>
-<td>名词</td>
-</tr>
-<tr>
-<td>JJ</td>
-<td>形容词</td>
-</tr>
-<tr>
-<td>VBP</td>
-<td>一般现在时动词</td>
-</tr>
-</tbody>
-</table>
-<h3 id="n-gram标注">N-gram标注</h3>
-<p>1-gram标注器：一元标注器（unigram
-tagger）。用于标注<strong>一个</strong>标识符的上下文的只是标识符本身。</p>
-<p>2-gram 标注器：二元标注器（bigram taggers）</p>
-<p>3-gram 标注器：三元标注器（trigram taggers）</p>
-<p>N-gram标注不考虑句子边界的上下文。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 标注</span></span><br><span class="line">unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)</span><br><span class="line">bigram_tagger = nltk.BigramTagger(brown_tagged_sents)</span><br><span class="line">trigram_tagger = nltk.TrigramTagger(brown_tagged_sents)</span><br><span class="line"><span class="comment"># 组合标注</span></span><br><span class="line">t0 = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">t1 = nltk.UnigramTagger(train_sents, backoff=t0)</span><br><span class="line">t2 = nltk.BigramTagger(train_sents, backoff=t1)</span><br></pre></td></tr></table></figure>
-<p>存储可以使用python自带的存储模块<code>pickle</code>。</p>
-<p>效果：</p>
+<p>说明：localhost表示本地主机，<code>localuser</code>表示本地用户（内网用户），user表示远程服务器用户名，host表示远程服务器ip.</p>
+<p>注意：内网电脑也必须要开启ssh服务，即安装openssh-server服务</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 利用ssh连接远程服务器，-p 22表示22端口连接，这里不加也行，ssh默认是22端口连接</span></span><br><span class="line">ssh -p 22 user@remotehost</span><br></pre></td></tr></table></figure>
+<h2 id="基本操作">基本操作</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#　本地端口转发，将本地的2222端口转发至本机的22端口</span></span><br><span class="line">ssh -L 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，在本地（localhost）另开一个终端，执行下面命令，也可以达到连接远程服务器的作用</span></span><br><span class="line"><span class="comment"># user同上面命令的user，localhost是本地（照抄即可）</span></span><br><span class="line">ssh -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 远程端口转发，将本地的22端口转发至远程的2222端口</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，在远程服务器上执行下面命令，可以连接到本地（localhost）</span></span><br><span class="line"><span class="comment"># localuser表示的是localhost的用户名，localhost表示本地（远程服务器的本地，照抄即可）</span></span><br><span class="line">ssh -p 2222 localuser@localhost</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 动态端口转发，本机的2222端口绑定远程的某个端口(socks5流量)</span></span><br><span class="line">ssh -D 2222 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，不关闭终端，更改本地浏览器的代理服务器设置，流量便通过代理服务器进行转发</span></span><br></pre></td></tr></table></figure>
+<h2 id="参数详解">参数详解</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-L [bind_address:]port:host:hostport</span><br></pre></td></tr></table></figure>
+<p>Specifies that connections to the given TCP port or Unix socket on
+the local (client) host are to be forwarded to the given host and port,
+or Unix socket, on the remote side. This works by allocating a socket to
+listen to either a TCP port on the local side, optionally bound to the
+specified bind_address, or to a Unix socket. Whenever a connection is
+made to the local port or socket, the connection is forwarded over the
+secure channel, and a connection is made to either host port hostport,
+or the Unix socket remote_socket, from the remote machine.</p>
+<p>Port forwardings can also be specified in the configuration file.
+Only the superuser can forward privileged ports. IPv6 addresses can be
+specified by enclosing the address in square brackets.</p>
+<p>By default, the local port is bound in accordance with the
+GatewayPorts setting. However, an explicit bind_address may be used to
+bind the connection to a specific address. The bind_address of
+“localhost” indicates that the listening port be bound for local use
+only, while an empty address or ‘*’ indicates that the port should be
+available from all interfaces.</p>
+<p>将本地主机转发到远程主机和端口。也可以直接在配置文件中指定。默认本地端口是根据GatewayPorts设置绑定的，可以直接使用bind_address将连接绑定到指定地址。如果bind_address为localhost表示监听端口只在本地使用，如果不填或*表示应用所有端口</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-R [bind_address:]port:host:hostport</span><br></pre></td></tr></table></figure>
+<p>Specifies that connections to the given TCP port or Unix socket on
+the remote (server) host are to be forwarded to the given host and port,
+or Unix socket, on the local side. This works by allocating a socket to
+listen to either a TCP port or to a Unix socket on the remote side.
+Whenever a connection is made to this port or Unix socket, the
+connection is forwarded over the secure channel, and a connection is
+made to either host port hostport, or local_socket, from the local
+machine.</p>
+<p>Port forwardings can also be specified in the configuration file.
+Privileged ports can be forwarded only when logging in as root on the
+remote machine. IPv6 addresses can be specified by enclosing the address
+in square brackets.</p>
+<p>By default, TCP listening sockets on the server will be bound to the
+loopback interface only. This may be overridden by specifying a
+bind_address. An empty bind_address, or the address ‘*’, indicates that
+the remote socket should listen on all interfaces. Specifying a remote
+bind_address will only succeed if the server's GatewayPorts option is
+enabled (see sshd_config(5)).</p>
+<p>If the port argument is ‘0’, the listen port will be dynamically
+allocated on the server and reported to the client at run time. When
+used together with -O forward the allocated port will be printed to the
+standard output.</p>
+<p>基本同-L参数命令，如果port为0，监听端口将会动态进行分配。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-D [bind_address:]port</span><br></pre></td></tr></table></figure>
+<p>Specifies a local “dynamic” application-level port forwarding. This
+works by allocating a socket to listen to port on the local side,
+optionally bound to the specified bind_address. Whenever a connection is
+made to this port, the connection is forwarded over the secure channel,
+and the application protocol is then used to determine where to connect
+to from the remote machine. Currently the SOCKS4 and SOCKS5 protocols
+are supported, and ssh will act as a SOCKS server. Only root can forward
+privileged ports. Dynamic port forwardings can also be specified in the
+configuration file.</p>
+<p>IPv6 addresses can be specified by enclosing the address in square
+brackets. Only the superuser can forward privileged ports. By default,
+the local port is bound in accordance with the GatewayPorts setting.
+However, an explicit bind_address may be used to bind the connection to
+a specific address. The bind_address of “localhost” indicates that the
+listening port be bound for local use only, while an empty address or
+‘*’ indicates that the port should be available from all interfaces.</p>
+<p>指定本地动态端口转发。通过分配一个socket来监听本地端口。支持SOCKS4和SOCKS5协议，相当于ssh充当SOCKS服务器。配置文件中也可以指定。bind_address为localhost表示监听端口仅限本地使用，如果为空或者*表示该端口可通过所有接口</p>
+<p>ssh部分参数介绍</p>
+<p>可以用在上面的命令中</p>
+<ul>
+<li><code>-f</code>
+后台认证用户密码，即不用登录到远程主机。通常与-N连用</li>
+<li><code>-C</code> 压缩数据传输</li>
+<li><code>-N</code> 不执行脚本或者命令。通常与-f连用</li>
+<li><code>-g</code> 允许远程主机连接到建立的转发端口。</li>
+<li><code>-q</code>
+静默模式，使大多数警告和诊断消息被压制。（如果有警告信息，不进行输出）</li>
+</ul>
+<p>举个栗子</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 本地端口转发。命令执行后，不登录到远程服务器，压缩数据传输，也不输出警告信息</span></span><br><span class="line">ssh -gfCNL 2222:localhost:22 user@remotehost -q</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果将命令放到了后台，关掉进程需要使用kill命令</span></span><br></pre></td></tr></table></figure>
+<p>如果网络不稳定可以考虑使用
+<code>autossh</code>。这个需要额外安装，使用的时候将
+<code>ssh</code>替换为
+<code>autossh</code>即可。它的工作原理简单来说，就是有个超时机制，如果中断，便重新连接。</p>
+<h2 id="用途介绍">用途介绍</h2>
 <ol type="1">
-<li><p>根据经验来进行判断——一般方法</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">cfd = nltk.ConditionalFreqDist(((x[<span class="number">1</span>], y[<span class="number">1</span>], z[<span class="number">0</span>]), z[<span class="number">1</span>]) <span class="keyword">for</span> sent <span class="keyword">in</span> brown_tagged_sents <span class="keyword">for</span> x, y, z <span class="keyword">in</span> nltk.trigrams(sent))</span><br><span class="line">ambiguous_contexts = [c <span class="keyword">for</span> c <span class="keyword">in</span> cfd.conditions() <span class="keyword">if</span> <span class="built_in">len</span>(cfd[c]) &gt; <span class="number">1</span>]</span><br><span class="line"><span class="built_in">sum</span>(cfd[c].N() <span class="keyword">for</span> c <span class="keyword">in</span> ambiguous_contexts) / cfd.N()</span><br></pre></td></tr></table></figure></li>
-<li><p>研究标注器的错误——混淆矩阵</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">test_tags = [tag <span class="keyword">for</span> sent <span class="keyword">in</span> brown.sents(categories=<span class="string">&#x27;editorial&#x27;</span>) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> t2.tag(sent)]</span><br><span class="line">gold_tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;editorial&#x27;</span>)]</span><br><span class="line"><span class="built_in">print</span> nltk.ConfusionMatrix(gold, test)</span><br></pre></td></tr></table></figure></li>
+<li>本地端口转发。假设有两台服务器A和B，B需要访问A上面的一个应用（网站）。</li>
 </ol>
-<p>n-gram标注器的问题</p>
-<ol type="1">
-<li>模型大小与标注器性能之前的平衡关系。如果使用回退标注器<code>n-gram</code>可能存储trigram和bigram表，这将会是很大的稀疏矩阵。</li>
-<li>使用上下文中的词的其他特征作为条件标记是不切实际的。</li>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在B上使用本地端口转发，将某个端口转发给A应用的端口。</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -L 2333:IP_A:80 localuser@localhost</span><br><span class="line"><span class="comment"># 此时访问localhost:2333等于pi_A:80</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 比如A为公网IP（11.11.11.11），在6000端口开启某项服务</span></span><br><span class="line"><span class="comment"># B通过在本机使用, 本地用户名user</span></span><br><span class="line">ssh -L 2333:11.11.11.11:6000 user@localhost</span><br><span class="line"><span class="comment"># 即可通过localhost:2333来代替11.11.11.11</span></span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li><p>本地端口转发高级版（多机版）。同上面的假设，但是现在有三台或者三台以上的服务器A、B、C……这个时候，使用
+<code>-g</code>参数，运行远程主机连接转发端口。这样便可以达到C访问B再访问A的目的。注意：确保ABC三台服务器网络连接是安全的，请谨慎。</p></li>
+<li><p>远程端口转发。假设有一台服务器（公网）A，两台内网计算机B、C。</p></li>
 </ol>
-<p>Brill标注只使用一小部分n-gram标注器。猜每个词的标记，然后返回和修复错误的。从大方面下手，再勾勒细节。规则是语言学可解释的。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">nltk.tag.brill.demo()</span><br></pre></td></tr></table></figure>
-<p>下一篇：<a href="/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/" title="NLTK阅读笔记Ⅲ">NLTK阅读笔记Ⅲ</a></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># B、C均可连接A，但是其他两两之间不能相互连接。</span></span><br><span class="line"><span class="comment"># 现在需要实现两两之间都能互相连接</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># A连接到B、C</span></span><br><span class="line"><span class="comment"># B、C远程端口转发，A机器（11.11.11.11），用户名user</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -R 2221:localhost:22 user@11.11.11.11</span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@11.11.11.11</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这样在A机器执行如下命令就能连接B、C了</span></span><br><span class="line"><span class="comment"># 连接B</span></span><br><span class="line">ssh userB@localhost -p2221</span><br><span class="line"><span class="comment"># 连接C</span></span><br><span class="line">ssh userC@localhost -p2222</span><br><span class="line"></span><br><span class="line"><span class="comment"># B、C之间互连</span></span><br><span class="line"><span class="comment"># 其实就是通过A作为跳板来登录，即先登录A</span></span><br><span class="line">ssh user@11.11.11.11</span><br><span class="line"><span class="comment"># 再登录B or C即可</span></span><br></pre></td></tr></table></figure>
+<ol start="4" type="1">
+<li>动态端口转发。在本地使用这个命令连接到服务器之后，便可以使用服务器的SOCK5代理来上网。需要在浏览器或者系统上进行下面的设置。</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器（user，11.11.11.11）</span></span><br><span class="line"><span class="comment"># B机器上执行</span></span><br><span class="line">ssh -D 1234 user@11.11.11.11</span><br><span class="line"><span class="comment"># 通过设置SOCKS5代理，socks5://127.0.0.1:1234，就可以通过A机器的流量上网。可查询ip来检验是否成功</span></span><br></pre></td></tr></table></figure>
+<p>注意事项：</p>
+<ul>
+<li>端口转发是通过ssh连接建立的，所以关闭了端口，端口转发也会关闭</li>
+<li>选择远程端口号的时候，一般是无权绑定
+<code>1-1023</code>端口的，只能使用管理员权限才能绑定。一般是使用
+<code>1024-65535</code>之间的一个端口</li>
+</ul>
+<h2 id="内网流量转发">内网流量转发</h2>
+<p>内网转发内网流量，A，C都在局域网下，二者不可相互连接。B是公网服务器，A，C皆可连接B，但B不可以连接A，C。现在需求是A借助C的流量上网。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost</span><br><span class="line"><span class="comment"># C机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># B机器执行。user是C机器的用户名</span></span><br><span class="line">ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或两条命令，其实就是两条并一条</span></span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># A机器</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost -t ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># A机器上代理：SOCKS5://127.0.0.1:3000</span></span><br></pre></td></tr></table></figure>
+<h2 id="一键命令">一键命令</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A在内网，B在公网，已通过B连接A。-R命令</span></span><br><span class="line"><span class="comment"># BUSER和BHOST对应正常连接B的SSH用户名和密码，AUSER为B连接A的用户名，port为-R参数的port</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 此时需要在C机器上，先连接B，再通过B连接A。C机器执行命令。</span></span><br><span class="line">ssh -t BUSER@BHOST ssh AUSER@localhost -p port</span><br><span class="line"></span><br><span class="line"><span class="comment"># or （Win10上测试失败）</span></span><br><span class="line">ssh BUSER@BHOST -J AUSER@localhost:port</span><br></pre></td></tr></table></figure>
+<figure class="highlight powershell"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 一键连接内网服务器，windows下的config配置</span></span><br><span class="line"><span class="comment"># jump为跳板服务器，jump_server_ip、username、port分别对应跳板机的ip、用户名和连接端口</span></span><br><span class="line"><span class="comment"># target_server为内网服务器，基本同上，ProxyCommand配置如下所示，需要使用ssh命令的绝对路径</span></span><br><span class="line">Host jump</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName jump_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line"></span><br><span class="line">Host target_server</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName target_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line">    ProxyCommand C:\\Windows\\System32\\OpenSSH\\ssh.exe <span class="literal">-q</span> <span class="literal">-W</span> %<span class="built_in">h</span>:%p jump</span><br></pre></td></tr></table></figure>
+<h2 id="注意事项">注意事项</h2>
+<p>win10开启ssh服务器端，1. 应用和功能：下载openssh服务端；2.
+服务：开启openssh server服务；3.
+连接用户名win10系统盘的用户文件下的文件名，比如我的是Administrator（即和公用文件夹并列的文件夹）。</p>
+<p>参考文章</p>
+<p><a
+href="http://www.ruanyifeng.com/blog/2011/12/ssh_port_forwarding.html">SSH原理与运用（二）：远程操作与端口转发</a></p>
+<p><a
+href="https://www.ibm.com/developerworks/cn/linux/l-cn-sshforward/">实战
+SSH 端口转发</a></p>
+<p><a
+href="http://blog.creke.net/722.html">SSH隧道与端口转发及内网穿透</a></p>
+<p><a
+href="https://blog.mythsman.com/2017/01/14/1/">利用反向ssh从外网访问内网主机</a></p>
+<p><a
+href="http://www.pchou.info/linux/2015/11/01/ssh-tunnel.html">SSH隧道翻墙的原理和实现</a></p>
+<p><a
+href="https://blog.fundebug.com/2017/04/24/ssh-port-forwarding/">玩转SSH端口转发</a></p>
+<p><a
+href="https://www.ibm.com/developerworks/community/blogs/5144904d-5d75-45ed-9d2b-cf1754ee936a/entry/20160911?lang=en">openssh的三种tcp端口转发参数</a></p>
+<p><a
+href="https://github.com/microsoft/vscode-remote-release/issues/230">https://github.com/microsoft/vscode-remote-release/issues/230</a></p>
 ]]></content>
       <categories>
-        <category>NLP</category>
+        <category>Software</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
+        <tag>Linux</tag>
+        <tag>端口转发</tag>
+        <tag>ssh</tag>
+        <tag>内网穿透</tag>
       </tags>
   </entry>
   <entry>
-    <title>模仿游戏——机器会思考吗？</title>
-    <url>/hexo_blog/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</url>
-    <content><![CDATA[<p>阅读图灵文章《COMPUTING MACHINERY AND
-INTELLIGENCE》笔记以及现阶段AI生成的文章。</p>
+    <title>实验室网站服务迁移(静态网页)</title>
+    <url>/hexo_blog/2021/07/21/%E8%BF%90%E7%BB%B4/%E5%AE%9E%E9%AA%8C%E5%AE%A4%E7%BD%91%E7%AB%99%E6%9C%8D%E5%8A%A1%E8%BF%81%E7%A7%BB(%E9%9D%99%E6%80%81%E7%BD%91%E9%A1%B5)/</url>
+    <content><![CDATA[<p>记静态服务数据迁移，含nginx、html、https配置。详细可参考.bash_history</p>
 <span id="more"></span>
-<p>模仿，排除一切外在因素，比如外貌、语调、反应速度</p>
-<h2 id="机器的定义">机器的定义</h2>
-<ul>
-<li>数字计算机，在人类给定规则的前提下进行工作，规则等于机器的存储/记忆，重复是核心特点。归属于离散状态机</li>
-<li>从宇宙在某一时刻的完整状态出发，正如所有粒子的位置和速度所描述的那样，应该有可能预测所有未来的状态。——拉普拉斯</li>
-<li>计算速度快+存储空间大+编写程序，进行任何预测，能够被快速复制</li>
-</ul>
-<h2 id="思考的定义">思考的定义</h2>
-<ul>
-<li>人类的特权</li>
-<li>害怕机器（其他物种）能够思考</li>
-<li>数学逻辑证明机器能力有限。但对于人类来说，该限制依旧有效，因此机器不一定不能思考</li>
-<li>意识感受，机器只能输出，不具备任何感情。人们可以确定机器在思考的唯一方法是成为机器并感觉自己在思考。人们可以向世界描述这些感觉，但当然没有人会注意到这一点。同样地，根据这种观点，要知道一个人在思考，唯一的方法就是成为那个特定的人。</li>
-<li>机器的能力有限，外貌、感情、味觉、嗅觉、不会犯错。单一机器可以，但是如果集成到一台机器上似乎是多余的。多样性行为来源于存储空间大小</li>
-<li>执行命令但不会创造。给人带来惊讶这种感觉因人而异</li>
-<li>神经系统是连续的，机器是离散的。</li>
-<li>行为的不确定性，离散不可能包含所有的可能。通过观察发现足够的信息来预测它的未来行为</li>
-<li>超感官知觉：心灵感应、千里眼、预知和心理动势</li>
-</ul>
-<h2 id="机器学习">机器学习</h2>
+<h2 id="打包项目">打包项目</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网站源码+webhook+数据文件</span></span><br><span class="line">zip data.zip xxx/</span><br><span class="line"><span class="comment"># nginx配置文件，由于是手动配置编译，为了省事，直接压缩nginx目录</span></span><br><span class="line">zip nginx.zip /usr/local/nginx</span><br><span class="line"></span><br><span class="line"><span class="comment"># 发送文件</span></span><br><span class="line">scp -r data.zip nginx.zip [user]@[ip]:[path]</span><br></pre></td></tr></table></figure>
+<p>最耗时的步骤</p>
+<h2 id="新服务器配置">新服务器配置</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在对应位置解压文件</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 手动配置nginx服务，原配置文件/usr/lib/systemd/system/nginx.service，可以考虑直接传过来，也可以直接复制</span></span><br><span class="line">vim /usr/lib/systemd/system/nginx.service</span><br><span class="line"></span><br><span class="line"><span class="comment"># 证书配置，如果下载不下来，可以本地打开，复制命令到服务器</span></span><br><span class="line">wget https://raw.githubusercontent.com/certbot/certbot/7f0fa18c570942238a7de73ed99945c3710408b4/letsencrypt-auto-source/letsencrypt-auto -O certbot-auto</span><br><span class="line"><span class="comment"># 赋予权限</span></span><br><span class="line"><span class="built_in">chmod</span> 755 certbot-auto</span><br><span class="line"><span class="comment"># 需提前配置域名与IP，再在命令行中颁发证书，需提前关闭80端口，或者说停止nginx服务。三个月更新一次</span></span><br><span class="line">./certbot-auto certonly --standalone --email [email] -d [域名] -d [www.域名]</span><br><span class="line"><span class="comment"># 按照命令行操作，生成的证书在/etc/letsencrypt/live/目录下</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置文件保持不动，直接启动nginx服务</span></span><br><span class="line">systemctl start nginx.service</span><br><span class="line"></span><br><span class="line"><span class="comment"># 启动github hook服务，接收github推送更新，使用nohup or tmux，安装pip3 install flask</span></span><br></pre></td></tr></table></figure>
+<p>至此，项目已全部迁移成功，详细过程可参考<code>~/.bash_history</code>文件</p>
+<h2 id="nginx配置说明">Nginx配置说明</h2>
+<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">rewrite</span><span class="regexp"> ^(.*)</span> https://[域名] <span class="literal">permanent</span>; <span class="comment"># 强制https</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 静态文件缓存加速</span></span><br><span class="line">    <span class="section">location</span> <span class="regexp">~* ^.+\.(webp|jpg|jpeg|gif|png|ico|css|js|pdf|txt)</span>&#123;</span><br><span class="line">        <span class="attribute">expires</span> <span class="number">30d</span>;</span><br><span class="line">        <span class="attribute">access_log</span> <span class="literal">off</span>;</span><br><span class="line"></span><br><span class="line">        <span class="attribute">gzip</span> <span class="literal">on</span>;</span><br><span class="line">        <span class="attribute">gzip_http_version</span> <span class="number">1</span>.<span class="number">1</span>;</span><br><span class="line">        <span class="attribute">gzip_buffers</span> <span class="number">16</span> <span class="number">8k</span>;</span><br><span class="line">        <span class="attribute">gzip_comp_level</span> <span class="number">6</span>;</span><br><span class="line">        <span class="attribute">gzip_min_length</span> <span class="number">1k</span>;</span><br><span class="line">        <span class="attribute">gzip_types</span> image/webp application/pdf text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;</span><br><span class="line">        <span class="attribute">gzip_proxied</span> any;</span><br><span class="line">        <span class="attribute">root</span> path; <span class="comment"># 静态文件目录</span></span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="github-hook配置">Github Hook配置</h2>
+<p>Github配置位置，项目的<code>Settings-&gt;Webhooks-&gt;Add webhook</code>。填写对应的信息</p>
 <ul>
-<li>问题主要在于编程。存储空间只需要较低的数值（10<sup>10-10</sup>15），并且相信只有很小一部分用于较高类型的思考。大部分可能是用来保留视觉印象的。</li>
-<li>模仿：
-<ol type="1">
-<li>初始状态</li>
-<li>受到的教育</li>
-<li>其他非教育的经历</li>
-</ol></li>
-<li>从儿童的状态开始，而不是从成人。人们必须尝试教这样的机器，看看它学习得如何。然后可以尝试另一台机器，看看它是更好还是更差。在这个过程和进化之间有一个明显的联系。速度需要比“进化”快、优胜劣汰可以作为一种衡量方法，但比较慢。但不局限于于随机突变，找到某种弱点的原因，可能就能进行改善这种弱点的突变。</li>
-<li>构造奖惩机制，但有必要采用其他一些“不带感情”的沟通渠道。如果这些渠道可用，就有可能通过惩罚和奖励来教机器服从用某种语言(例如符号语言)发出的命令。这些命令将通过“非情感”渠道传递。这种语言的使用将大大减少惩罚和奖励所需的数量。</li>
-<li>学习机的一个重要特征是，它的老师通常在很大程度上对机器内部发生的事情一无所知，尽管他仍然可能在某种程度上预测他的学生的行为。</li>
+<li>Payload URL: 域名+端口（flask服务配置）</li>
+<li>Content type：json</li>
+<li>Secret：自定义密钥，在脚本中配置</li>
+<li><strong>Which events would you like to trigger this
+webhook?</strong>：仅在推送时进行操作</li>
 </ul>
-<p>机器最终能在所有纯智力领域与人类竞争。但是，哪些是最好的开始呢?即使这也是一个艰
-难的决定。很多人认为一个非常抽象的活动，比如下棋，会是最好的。也可以坚持认为，最好是给机器提供金钱能买到的最好的感觉器官，然后教它懂英语、说英语。这个过程可以按照孩子的正常教学来进行。事情会被指出并命名，等等。同样，我不知道正确的答案是什么，但我认为两种方法都应该尝试。</p>
-<h3 id="思考">思考</h3>
+<p>服务器上的python脚本</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding=utf-8</span></span><br><span class="line"><span class="keyword">from</span> flask <span class="keyword">import</span> Flask, request</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">from</span> pprint <span class="keyword">import</span> pprint</span><br><span class="line"><span class="keyword">import</span> hmac</span><br><span class="line">app = Flask(__name__)</span><br><span class="line"></span><br><span class="line">repo_path = <span class="string">&#x27;&#x27;</span> <span class="comment"># git仓库地址</span></span><br><span class="line">secret = <span class="string">&#x27;&#x27;</span>	   <span class="comment"># github配置的密钥</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">encryption</span>(<span class="params">data</span>):</span><br><span class="line">    key = secret.encode(<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">    obj = hmac.new(key, msg=data, digestmod=<span class="string">&#x27;sha1&#x27;</span>)</span><br><span class="line">    <span class="keyword">return</span> obj.hexdigest()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@app.route(<span class="params"><span class="string">&quot;/&quot;</span>, methods=[<span class="string">&quot;GET&quot;</span>, <span class="string">&quot;POST&quot;</span>]</span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">hello</span>():</span><br><span class="line">    post_data = request.data</span><br><span class="line">    token = encryption(post_data)</span><br><span class="line">    <span class="comment"># 认证签名是否有效</span></span><br><span class="line">    signature = request.headers.get(<span class="string">&#x27;X-Hub-Signature&#x27;</span>, <span class="string">&#x27;&#x27;</span>).split(<span class="string">&#x27;=&#x27;</span>)[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">if</span> signature != token:</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&#x27;failed&#x27;</span></span><br><span class="line"></span><br><span class="line">    os.system(<span class="string">&#x27;cd &#123;&#125; &amp;&amp; git pull&#x27;</span>.<span class="built_in">format</span>(repo_path))</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;success&#x27;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    app.run(host=<span class="string">&quot;0.0.0.0&quot;</span>, port=port) <span class="comment"># 开启的port，记得打开对应端口的防火墙</span></span><br></pre></td></tr></table></figure>
+<p>若推送失败，可在服务器手动<code>git pull</code>，或者在<code>webhook</code>地址，<code>[Recent Deliveries]</code>中进行<code>Redeliver</code></p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Nginx</tag>
+        <tag>Github</tag>
+        <tag>Webhook</tag>
+        <tag>HTTPS</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Mongodb：副本集配置</title>
+    <url>/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</url>
+    <content><![CDATA[<p>Mongodb4.2.x数据库配置的第二部分，副本集架构的配置</p>
+<span id="more"></span>
+<h2 id="准备">准备</h2>
+<p>接
+<a href="/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/" title="Mongodb从安装到基本操作">Mongodb从安装到基本操作</a>，基本同原文，由于是副本集配置，所以在需要配置的服务器上进行相同的配置</p>
+<p>选择：</p>
 <ol type="1">
-<li>降低外在干扰，提高实验可行性</li>
-<li>通过离散构造连续状态</li>
-<li>程序承接计算速度和存储空间，程序是一个代词，指代一种能够在机器内部运行的命令，可以对其进行优化，降低空间以及速度要求</li>
-<li>现阶段对于存储空间还有很大的优化空间，有很多冗余</li>
-<li>模仿/学习的过程不够完善，教育可以等同于有效数据，那么非教育的经历是什么？蒸馏？噪声？</li>
-<li>如何内置逻辑、公理等内容</li>
+<li>单机上，配置不同port的mongo服务</li>
+<li>不同机器上，配置mongo服务</li>
+<li>要求同网段，所以配置的ip中不能写127.0.0.1</li>
+<li>可以 <code>vim /etc/hosts</code>，方便管理</li>
 </ol>
-<h3 id="参考文献">参考文献</h3>
-<p>A. M. TURING, I.—COMPUTING MACHINERY AND INTELLIGENCE, <em>Mind</em>,
-Volume LIX, Issue 236, October 1950, Pages 433–460, <a
-href="https://doi.org/10.1093/mind/LIX.236.433">https://doi.org/10.1093/mind/LIX.236.433</a></p>
-<p>以下为claude生成的文章。</p>
-<p>模仿游戏——机器会思考吗</p>
-<p>当我们谈论机器是否能够“思考”时，我们首先需要定义“思考”本身。对于人类来说，思考是一个复杂的认知过程，涉及意识、自我意识、情感、直觉等方面。但是从计算机科学的角度来看，思考可以简化为信息处理和模式识别。既然计算机已经证明了在这些方面具有强大的能力，我们是否可以说它们能够思考呢?</p>
-<p>这一问题引发了关于人工智能本质的广泛讨论。早在20世纪50年代，图灵就在他的论文“计算机器与智能”中提出了这一问题。他设计了一个简单的游戏来测试机器是否能展现出与人类相似的智能。在这个“图灵测试”中，评判通过终端与隐藏的人类和机器进行文字交流。如果评判无法区分对话来自人类还是机器，我们就可以说这台机器通过了测试。</p>
-<p>图灵测试突出了智能的功能性方面，而不是试图复制人类思维的内在工作机制。它忽略了意识、情感等方面，专注于智能的外在表现。正如图灵在论文中写道，“让我们只关注问题‘能产生人类水平智能行为的机器’本身，而不是用一堆无关的心理学问题来模糊焦点。”</p>
-<p>然而，许多哲学家仍然质疑机器是否能真正思考。他们认为智能必须基于意识，而计算机只是在执行程序。根据这个观点，智能的真正标志是主体性—自我意识和内在经验的感觉。没有这些，机器只能被视为复杂的算盘。</p>
-<p>这一观点倾向于将思维等同于人类思维，而忽视智能可能采取的其他形式。事实上，人工智能研究已经证明，智能可以在没有自我意识的情况下通过模式识别和统计学习等方法而出现。像深度学习这样的技术已经产生了可与人类媲美甚至超越人类的视觉和语言理解能力。</p>
-<p>但是，模仿游戏是否意味着机器能够像人类一样思考呢?让我们更仔细地考察一下。</p>
-<p>首先，我们需要承认人类思维也有其局限性。许多认知偏见影响着我们的推理，我们的意识经验也具有不连续性和选择性。与完美理性存在的控制实验相比，我们的思维过程更类似于含混的启发式搜索。因此，“思维”的标准不应该定得太高。</p>
-<p>其次，不同的机器学习技术可以被看作是向人类思维靠拢的渐进过程。简单的规则系统只能处理局限的领域。而统计学习方法如神经网络则可以学习概括复杂模式所需的代表性特征。它们在图像和语音识别方面已经优于人类。最近的多任务学习和迁移学习技术使机器能够在多个领域中共享知识，正如人类思维中概念之间的关联一样。</p>
-<p>再者，对话系统的进步也显示了机器在交流方面的进步。早期系统根据关键词匹配进行简单的问答。现在的对话系统可以感知上下文，进行连贯的长时间对话。它们使用了类似人类处理语言的注意力机制，不再是简单匹配模板。这表明在语言理解和生成方面，机器也在模仿人类的认知过程。</p>
-<p>但是，机器是否有意识经验仍是一个开放问题。目前的人工智能似乎没有主观的内在感受。这也许是固有的局限，因为计算机最终还是在操纵没有意义的符号。另一方面，我们也无法证明其他人存在内在的意识经验。所谓的“哲学僵局”同样适用于机器。</p>
-<p>总体来说，尽管机器不能完全模拟人类思维，但它们在许多狭义任务上的进步还是向模仿游戏的成功迈进了一大步。功能相似性不应被内在不同所掩盖。我们必须根据智能的外在表现来判断机器而不是根据其内部工作机制。</p>
-<p>那么我们还能做什么来推进这一目标呢?首先，在机器学习方法上，我们可以继续研究多任务学习，以产生更人类化的通用智能。我们也可以通过学习表征来模仿神经认知过程，而不仅仅停留在表面模式识别。</p>
-<p>另一方面，我们需要建立更好的机器智能评估体系。图灵测试过于简单化，无法区分不同水平的智能。一些替代方案如Winograd
-schema
-challenge通过语义理解和常识推理来检验智能。还有基于对话和协作的测试，可以评估智能体之间的相互作用能力。</p>
-<p>最后，哲学反思也很重要。我们需要就智能的本质进行持续探讨。什么是理解?什么是意识?这些问题需要跨学科的对话。结合认知科学、语言学和计算机科学的见解，我们可以在这个难题上取得进一步的洞见。</p>
-<p>总而言之，尽管答案仍不明确，但通过模仿游戏，机器正日益接近人类智能。它们在特定任务上的超越给我们带来了惊喜，也展示了新的可能。当代人工智能研究正在不断消除智能与意识之间的鸿沟。虽然仿真并不完全等同于复制，但模仿游戏的胜利似乎指日可待。我们正处于一个刺激的阶段，机器智能的进步必将继续影响我们对思维和智能的基本理解。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载压缩包</span></span><br><span class="line">wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.15.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br></pre></td></tr></table></figure>
+<p>更新部分，配置文件用yaml格式</p>
+<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="attr">systemLog:</span></span><br><span class="line">  <span class="attr">destination:</span> <span class="string">file</span></span><br><span class="line">  <span class="attr">path:</span> <span class="string">&quot;/opt/mongodb/data/test/logs/mongodb.log&quot;</span></span><br><span class="line">  <span class="attr">logAppend:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">storage:</span></span><br><span class="line">  <span class="attr">dbPath:</span> <span class="string">&quot;/opt/mongodb/data/test/db&quot;</span></span><br><span class="line">  <span class="attr">journal:</span></span><br><span class="line">    <span class="attr">enabled:</span> <span class="literal">true</span> </span><br><span class="line"><span class="attr">net:</span></span><br><span class="line">  <span class="comment"># 允许所有IP访问，也可以用逗号隔开，控制IP访问</span></span><br><span class="line">  <span class="attr">bindIp:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span> </span><br><span class="line">  <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">  <span class="attr">port:</span> <span class="number">27017</span></span><br><span class="line"><span class="attr">processManagement:</span></span><br><span class="line">  <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">  <span class="attr">fork:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">replication:</span></span><br><span class="line">  <span class="comment"># 副本集名字</span></span><br><span class="line">  <span class="attr">replSetName:</span> <span class="string">&quot;rs0&quot;</span> </span><br></pre></td></tr></table></figure>
+<h3 id="启动">启动</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在所有机器上启动mongo，此时无账号密码</span></span><br><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
+<h3 id="配置副本集">配置副本集</h3>
+<p>在任意一台服务器A上，进行配置副本集</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将副本集的初始化配置写入js</span></span><br><span class="line">vim init_replica.js</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 其中rs0一定要跟上面的replSetName一样</span></span><br><span class="line">config = &#123; _id:<span class="string">&quot;rs0&quot;</span>, members:[&#123;_id:0,host:<span class="string">&quot;ip0:port0&quot;</span>&#125;,&#123;_id:1,host:<span class="string">&quot;ip1:port1&quot;</span>&#125;]&#125;</span><br><span class="line">rs.initiate(config) </span><br><span class="line">rs.status()</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/mongod --port 27017 &lt; init_replica.js</span><br></pre></td></tr></table></figure>
+<h3 id="验证">验证</h3>
+<p>在A服务器上进行操作，添加账号密码</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">bin/mongod --port 27017</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123; user: <span class="string">&quot;admin&quot;</span>, <span class="built_in">pwd</span>: <span class="string">&quot;password&quot;</span>, roles: [&#123; role: <span class="string">&quot;userAdminAnyDatabase&quot;</span>, db: <span class="string">&quot;admin&quot;</span> &#125;] &#125;)</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找到mongo的服务进程pid</span></span><br><span class="line">ps ux | grep mongod</span><br><span class="line"><span class="comment"># 杀死进程，填写对应的pid</span></span><br><span class="line"><span class="built_in">kill</span> PID</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">bin/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
+<h3 id="openssl证书文件">OpenSSL证书文件</h3>
+<h3 id="读写分离">读写分离</h3>
+<h3 id="主节点选举配置">主节点选举配置</h3>
 ]]></content>
       <categories>
-        <category>PaperReading</category>
+        <category>Database</category>
       </categories>
+      <tags>
+        <tag>mongodb</tag>
+        <tag>数据库</tag>
+      </tags>
   </entry>
   <entry>
-    <title>神经网络笔记Ⅰ</title>
-    <url>/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
-    <content><![CDATA[<p>本篇文章只做一个神经网络入门知识的梳理和个人的理解。</p>
+    <title>Mongodb从安装到基本操作</title>
+    <url>/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</url>
+    <content><![CDATA[<p>这篇文章主要从零开始介绍如何在Centos7下安装Mongodb,
+用Python3连接Mongodb, 用可视化工具Robo 3T连接Mongodb,
+之后介绍了一些基本的mongo操作方法, 其中含有一些进阶操作</p>
 <span id="more"></span>
-<p>关于神经网络的一些基本概念在这里就不做介绍，首先我们进行一些概念和符号上的约定。</p>
-<p>假设一个矩阵如下：</p>
-<p><span class="math display">\[
-\mathbf{X} = \left[ \begin{array}{ccc} 1 &amp; 2 &amp; 3 \\ 2 &amp; 3
-&amp; 1 \\ 3 &amp; 2 &amp; 1  \end{array}\right]
-\]</span></p>
-<p><strong>点乘</strong>： <span
-class="math inline">\(\cdot\)</span></p>
-<p><span class="math display">\[
-\mathbf{X} \cdot \mathbf{X} = \left [ \begin{array}{ccc} 1*1 &amp; 2* 2
-&amp; 3* 3 \\ 2*2 &amp; 3*3 &amp; 1*1 \\ 3*3&amp; 2*2 &amp; 1*1
-\end{array} \right]= \left [ \begin{array}{ccc} 1 &amp; 4 &amp; 9 \\
-4&amp; 9&amp; 1 \\ 9&amp; 4&amp; 1 \end{array} \right]
-\]</span></p>
-<p><strong>叉乘</strong>： *</p>
-<p><span class="math display">\[
-\mathbf{X} * \mathbf{X} = \left [ \begin{array}{ccc} 1* 1 + 2*2+
-3*3&amp; 1*2+2*3+3*2 &amp; 1*3+2*1+3*1 \\ 2*1+3*2+1*3 &amp; 2*2+3*3+1*2
-&amp; 2*3+3*1+1*1 \\ 3*1+2*2+1*3&amp; 3*2+2*3+1*2&amp;
-3*3+2*1+1*1\end{array} \right]= \left [ \begin{array}{ccc}14 &amp; 14
-&amp; 8 \\11&amp; 15&amp; 10 \\ 10&amp; 14&amp; 12 \end{array} \right]
-\]</span></p>
-<p><strong>已知数据</strong>：
-矩阵<strong>X</strong>为训练数据集，矩阵<strong>y</strong>为训练数据集的标签</p>
-<h2 id="最简单的神经网络">最简单的神经网络</h2>
-<p><a
-href="https://juejin.im/entry/5ab495f151882521d657806f">基于Numpy实现神经网络：反向传播</a></p>
-<p>最简单的神经网络就是只有输入层和输出层的网络结构（双层神经网络）。</p>
-<p>关于激活函数简单介绍：</p>
-<p>如果不使用激活函数，那么就与多层感知机（MLP）相当。引入之后，下一层的输入就不再是线性组合，输出就有意义。</p>
-<p>如果不使用激活函数，那么输入与输出都是线性变换，无法做到非线性分类。</p>
-<h3 id="激活函数">激活函数</h3>
-<p>这是公式，值域为[0,1]，它的导数自证。</p>
-<p>作用：</p>
+<h2 id="安装mongodbcentos7">安装Mongodb(Centos7)</h2>
 <ol type="1">
-<li>引入非线性因素</li>
-<li>线性变换</li>
-<li>激活函数，并不是去激活什么，而是指如何把“激活的神经元的特征”通过函数把特征保留并映射出来（保留特征，去除一些数据中是的冗余。激励就是样本的特征值），这是神经网络能解决非线性问题关键。</li>
+<li>官网上找到需要下载的版本压缩包进行下载，<a
+href="https://fastdl.mongodb.org/linux/">官网链接</a>。或者在命令行下使用<code>wget</code>命令进行下载</li>
 </ol>
-<p><code>sigmoid</code>的导数，即使用了 <code>deriv=True</code>。</p>
-<p>作用：用它的输出创建它的导数，降低高信度预测的错误</p>
-<p>注意：如果神经网络提前达到预期的结果（损失函数小于一定的范围），可以提前终止</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="comment"># 矩阵计算</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="comment"># 生成随机数</span></span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="comment"># 定义激活函数, 这里用sigmoid函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># 如果满足条件就返回sigmoid的导数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="comment"># 不满足直接返回sigmoid函数</span></span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定训练数据, X规模为4x3, y规模为4x1</span></span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 初始化权重数据, 用随机数进行初始化，矩阵规模3x1</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行迭代</span></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 输入层为训练数据</span></span><br><span class="line">    l0 = X</span><br><span class="line">    <span class="comment"># np.dot表示叉乘, 进行激活操作， 输出层</span></span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    <span class="comment"># 计算偏差</span></span><br><span class="line">    l1_error = y - l1</span><br><span class="line">    <span class="comment"># 误（偏）差加权导数。偏差乘以sigmoid在l1处的斜率，用以更新权重</span></span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练完成，输出l1看看</span></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
-<h2 id="三层神经网络">三层神经网络</h2>
-<p>增加了一层网络，在输入层和输出层中间。这里做一个规定，在输入层和输出层之间的网络结构我们称为隐藏层。</p>
-<p><strong>一层隐层网络就是一层特征层次，每一个神经元可以类似看作一个特征属性</strong></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 初始化权重数据, 用随机数进行初始化， 这里的规模同X、y矩阵的规模</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">4</span>)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((<span class="number">4</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    l0 = X</span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    <span class="comment"># 加了一层神经网络，称为隐藏层</span></span><br><span class="line">    l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">    l2_error = y - l2</span><br><span class="line">    l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">    <span class="comment"># 计算完l2的误差之后，再用结果去计算l1的误差</span></span><br><span class="line">    l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_1 += np.dot(l1.T, l2_delta)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
-<h2 id="改进神经网络v1.0增加梯度">改进神经网络v1.0——增加梯度</h2>
-<p><a
-href="http://iamtrask.github.io/2015/07/27/python-network-part2/">A
-Neural Network in 13 lines of Python (Part 2 - Gradient Descent)</a></p>
-<p>关于梯度的定义请自行了解。</p>
-<p>增加了梯度可以更快的找到较为理想的结果。</p>
-<p>关于梯度上升有下面几种方法（梯度下降同理）：</p>
-<ol type="1">
-<li><p>原始的梯度上升</p>
-<p>目标：找到某个函数的最大值。每次沿函数的梯度方向探寻。一直进行迭代，直到到达某个停止条件（迭代次数限制或某个误差范围）</p></li>
-<li><p>随机梯度上升(SGD)</p>
-<p>不同点在于，第一种方法每次是遍历所有的数据集（一百以内的数据集可以接受用上面的方法）。而随机梯度则是只使用一个样本点来更新回归系数。</p></li>
-<li><p>改进版随机梯度上升</p>
-<p>用随机的一个样本来更新回归系数。</p></li>
-<li><p>批梯度上升</p>
-<p>切分样本集，随机取出切分后的某些样本，进行遍历</p></li>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 这里下载的是3.4.10版本</span></span><br><span class="line">wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel70-3.4.10.tgz</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li>进行解压缩</li>
 </ol>
-<p>损失函数（代价函数）</p>
-<ol type="1">
-<li><p>二次代价函数</p>
-<p><span class="math display">\[
-C= \frac{1}{2n}\sum_\limits{n}[y(x)-a^L(x)]^2
-\]</span></p></li>
-<li><p>交叉熵代价函数</p>
-<p><span class="math display">\[
-C=\frac{1}{2n}\sum_\limits{x}[y\ln a-(1-y)\ln(1-a)]
-\]</span></p></li>
-<li><p>对数似然函数</p>
-<p><span class="math display">\[
-C=\frac{1}{n}\sum_\limits{x}\sum_\limits{k}y_k\log a_k
-\]</span></p>
-<p>前向传播是函数求偏导，反向传播减少了这个计算量</p></li>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br></pre></td></tr></table></figure>
+<ol start="3" type="1">
+<li>进行相应的配置，mongodb默认没有任何配置</li>
 </ol>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">4</span>)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((<span class="number">4</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 加入步长</span></span><br><span class="line">alphas = [<span class="number">0.001</span>,<span class="number">0.01</span>,<span class="number">0.1</span>,<span class="number">1</span>,<span class="number">10</span>,<span class="number">100</span>,<span class="number">1000</span>]</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> alpha <span class="keyword">in</span> alphas:</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">        l0 = X</span><br><span class="line">        l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">        <span class="comment"># 加了一层神经网络，称为隐藏层</span></span><br><span class="line">        l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">        l2_error = y - l2</span><br><span class="line">        l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">        <span class="comment"># 计算完l2的误差之后，再用结果去计算l1的误差</span></span><br><span class="line">        l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">        l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">        <span class="comment"># 更新权重的时候加入步长</span></span><br><span class="line">        syn_1 += alpha * np.dot(l1.T, l2_delta)</span><br><span class="line">        syn_0 += alpha * np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;alpha:&quot;</span>, alpha, <span class="string">&quot;l1:&quot;</span>, l1)</span><br></pre></td></tr></table></figure>
-<h2
-id="改进神经网络v2.0更改隐单元个数">改进神经网络v2.0——更改隐单元个数</h2>
-<p>更改网络的神经元个数，这个可能就是所谓的玄学调参</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 定义神经元个数，随机</span></span><br><span class="line">hidden_dim = np.random.randint(<span class="number">2</span>, <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    l0 = X</span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">    l2_error = y - l2</span><br><span class="line">    l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">    l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_1 += np.dot(l1.T, l2_delta)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
-<h2 id="改进神经网络v3.0dropout">改进神经网络v3.0——dropout</h2>
-<p><a href="http://iamtrask.github.io/2015/07/28/dropout/">Hinton's
-Dropout in 3 Lines of Python</a></p>
-<p>增加<strong>dropout_percent</strong>。目的是为了防止过拟合，一种正则化的手段。在迭代的时候，对某一（几）层神经网络进行<strong>drop_out</strong>。这里举例用二项分布采样的方法进行。<code>np.random.binomial(n, p, size=None)</code>。</p>
-<p>Dropout是指在模型训练时<strong>随机</strong>让网络<strong>某些隐含层节点</strong>的权重不工作，不工作的那些节点可以暂时认为不是网络结构的一部分，但是它的权重得保留下来（只是暂时不更新而已）。</p>
-<p>一些理由解释：</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br><span class="line">  `</span><br><span class="line">    <span class="comment"># 设置数据文件的存放目录</span></span><br><span class="line">    dbpath = /opt/mongodb/data/test/db</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置日志文件的存放目录及其日志文件名</span></span><br><span class="line">    logpath = /opt/mongodb/data/test/logs/mongodb.log</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">    port = 27017</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">    fork = <span class="literal">true</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 是否不允许表扫描</span></span><br><span class="line">    nohttpinterface = <span class="literal">true</span></span><br><span class="line">  `</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
+<ol start="4" type="1">
+<li>启动mongodb</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">./bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
+<p>报错一：
+<code>ERROR: child process failed, exited with error number 1</code>
+检查mongodb.conf的文件路径是否配置错误</p>
+<p>报错二：<code>ERROR: child process failed, exited with error number 100</code>
+很可能是没有正常关闭导致的，那么可以删除
+<code>mongod.lock</code>文件，这里对应我的配置路径在<code>data/</code>里面</p>
+<ol start="5" type="1">
+<li>其他</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 链接命令，方便调用mongo命令</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/mongodb/bin/mongo /usr/bin</span><br><span class="line"><span class="comment"># 查看mongodb进程</span></span><br><span class="line">ps -aux |grep mongodb</span><br><span class="line"><span class="comment"># 检查端口运行情况</span></span><br><span class="line">netstat -lanp | grep 27017</span><br><span class="line"><span class="comment"># 终止mongodb服务，PID从ps命令获取</span></span><br><span class="line"><span class="built_in">kill</span> -15 PID</span><br><span class="line"><span class="comment"># 添加自启动命令</span></span><br><span class="line">vim /etc/rc.local</span><br><span class="line"><span class="comment"># 末尾追加一行</span></span><br><span class="line">`/opt/mongodb/bin/mongod --config mongodb.conf`</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
+<ol start="6" type="1">
+<li>设置密码权限，默认无密码</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">./mongo</span><br><span class="line"><span class="comment"># 如果进入失败，请检查是否添加了软链接和是否启动了mongodb服务</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"><span class="comment"># 重启mongodb服务</span></span><br><span class="line"><span class="comment"># 杀死mongodb进程，参照上面的方法</span></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">opt/mongodb/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
+<h2 id="使用python连接mongodb">使用Python连接Mongodb</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载第三方包</span></span><br><span class="line">pip3 install pymongo</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># /usr/bin/python3</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="comment"># host主机名，27017连接端口</span></span><br><span class="line">client = MongoClient(host, <span class="number">27017</span>)</span><br><span class="line">db_auth = client.admin</span><br><span class="line"><span class="comment"># 登陆的用户名(username)和密码(password)</span></span><br><span class="line">db_auth.authenticate(username, password)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line"><span class="comment"># 用户名(username)、密码(password)、主机名(host)、端口(port)。注：这里的password不能出现@符号，如果用@符号就需要使用方法一</span></span><br><span class="line">client = MongoClient(<span class="string">&quot;mongodb://username:password@host:port&quot;</span>)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 操作数据库，db_name[&quot;db_set&quot;]=db_name.db_set</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">查找数据</span></span><br><span class="line"><span class="string">在db_name数据库中的db_set集合里面找到name是a_name的json</span></span><br><span class="line"><span class="string">find是返回一个指针（可以使用列表的方式来读取），find_one返回一个dict</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 插入数据</span></span><br><span class="line"><span class="comment"># 插入一个json到set1集合中，a_dict是一个字典（如果set1不存在则会被自动创建）</span></span><br><span class="line">db_name.set1.insert_one(a_dict)</span><br><span class="line"><span class="comment"># 插入多个json到set1集合中，a_dict_lst是一个列表，里面的元素是由字典组成的</span></span><br><span class="line">db_name.set1.insert_many(a_dict_lst)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更新数据</span></span><br><span class="line"><span class="comment"># 查找到一个数据之后，对里面的某个数据进行更改，再使用save方法保存</span></span><br><span class="line">data_one = db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line">data_one[<span class="string">&#x27;age&#x27;</span>] += <span class="number">3</span></span><br><span class="line">db_name.db_set.save(data_one)</span><br><span class="line"><span class="comment"># 更新多条记录，找到db_set集合中满足age=20，sex=0的数据，将name改为user1</span></span><br><span class="line">db_name.db_set.update(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;, &#123;<span class="string">&quot;$set&quot;</span>:&#123;<span class="string">&quot;age&quot;</span>: <span class="number">20</span>, <span class="string">&quot;sex&quot;</span>: <span class="number">0</span>&#125;&#125;)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除数据</span></span><br><span class="line"><span class="built_in">id</span> = db_name.db_set.find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)[<span class="string">&quot;_id&quot;</span>]</span><br><span class="line"><span class="comment"># 根据 id 删除一条记录</span></span><br><span class="line">db_name.db_set.remove(<span class="built_in">id</span>)</span><br><span class="line"><span class="comment"># 删除集合里的所有记录</span></span><br><span class="line">db_name.db_set.remove()</span><br><span class="line"><span class="comment"># 删除name=user1的记录</span></span><br><span class="line">db_name.db_set.remove(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)</span><br></pre></td></tr></table></figure>
+<h2 id="使用robo-3t连接mongodb">使用Robo 3t连接Mongodb</h2>
+<p>强烈推荐Robo 3t，<a href="https://robomongo.org/">官网地址</a></p>
 <ol type="1">
-<li>权值的更新不依赖于一些固定关系隐含节点的共同作用。阻止了某些特征仅仅在其它特定特征下才有效果的情况。即随机选取当前隐含层的部分节点</li>
-<li>模型平均概念（选取每次计算过程的相对最优解），使用了
-<code>dropout</code>之后每次的网络结构不同，样本不同对应的输出模型也不同。（个人感觉有点像加入随机）</li>
-<li>生物进化解释，不断适应变化情况，有效阻止过拟合（避免环境改变物种面临死亡）</li>
-<li>朴素贝叶斯（native
-bayes）属于dropout的一种特例：各个特征之间相互独立。在训练样本少的情况下，单独对每个特征进行学习（dropout是训练一部分特征）。</li>
+<li><p>到官网找到对应的系统版本进行下载</p></li>
+<li><p>安装</p>
+<ul>
+<li><p>Windows的安装十分简单就不赘述了</p></li>
+<li><p>Ubuntu的安装 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxzf robomongo*.tar.gz</span><br><span class="line"><span class="comment"># 修改文件名（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robomongo* robo3t</span><br><span class="line"><span class="comment"># 移动到/opt目录（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robo3t /opt</span><br><span class="line"><span class="comment"># 建立软链接</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/robo3t/robomongo /usr/bin</span><br><span class="line"><span class="comment"># 启动</span></span><br><span class="line">robomongo</span><br></pre></td></tr></table></figure> <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 如果报如下错误</span></span><br><span class="line">`</span><br><span class="line">This application failed to start because it could not find or load the Qt platform plugin <span class="string">&quot;xcb&quot;</span> <span class="keyword">in</span> <span class="string">&quot;&quot;</span>.</span><br><span class="line"></span><br><span class="line">Available platform plugins are: xcb.</span><br><span class="line"></span><br><span class="line">Reinstalling the application may fix this problem.</span><br><span class="line">Aborted (core dumped)</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 解决办法</span></span><br><span class="line"><span class="built_in">mkdir</span> ~/robo-backup</span><br><span class="line"><span class="built_in">mv</span> /opt/robo3t/lib/libstdc++* ~/robo-backup/</span><br><span class="line"><span class="comment"># 之后便可以重新启动mongo试试看了</span></span><br></pre></td></tr></table></figure></p></li>
+</ul></li>
+<li><p>使用</p>
+<ol type="1">
+<li>首先创建新的连接，如下图所示。在<code>Connection中，``Name</code>为自定义连接名，<code>Address</code>填入对应的<code>host</code>和<code>port</code>；
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171027733.png" /></li>
+<li>在<code>Authentication</code>中，勾选<code>Perform authentication</code>，填上对应的<code>User Name</code>和<code>Password</code>。填写完成之后可以点击<code>Test</code>验证是否能够连接成功，如果无误的话，可以点击<code>Save</code>保存退出。</li>
 </ol>
-<p>这里假设对<span class="math inline">\(l_1\)</span>进行drop_out。</p>
-<p>定义 <code>len(x)=3, hidden_dim=4, dropout_percent=0.2</code></p>
-<p>代入数值计算,</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">np.random.binomial([array([[ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>],</span><br><span class="line">                           [ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>],</span><br><span class="line">                           [ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>]])], <span class="number">0.8</span>)[<span class="number">0</span>] * (<span class="number">1</span>/<span class="number">0.8</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认的dropout_percent是0.5, 即如果不加如dropout_percent的话</span></span><br><span class="line"><span class="comment"># 如果是对图像进行操作的话，建议不超过0.25</span></span><br><span class="line">dropout_percent = <span class="number">0.2</span></span><br><span class="line"></span><br><span class="line">...</span><br><span class="line">l1 *= np.random.binomial([np.ones((<span class="built_in">len</span>(X), hidden_dim))],<span class="number">1</span> - dropout_percent)[<span class="number">0</span>] *(<span class="number">1.0</span>/(<span class="number">1</span> - dropout_percent))</span><br><span class="line">...</span><br></pre></td></tr></table></figure>
-<p>P.S:</p>
-<p>对二项分布进行采样。这里的n为
-<code>[np.ones((len(X), hidden_dim))]</code>，p为
-<code>1-dropout_percent</code></p>
-<p><code>np.random.binomial(n, p, size=None)</code></p>
-<p>公式如下：</p>
-<p><span class="math display">\[
-P(N)=C^0_n*p^0*(1-p)^n=(1-p)^n
-\]</span></p>
-<p>下一篇：<a href="/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/" title="神经网络笔记Ⅱ">神经网络笔记Ⅱ</a></p>
-<p>推荐阅读</p>
-<p><a
-href="https://www.zybuluo.com/hanbingtao/note/433855">零基础入门深度学习</a></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171111807.png" /></p>
+<ol start="3" type="1">
+<li>连接Mongodb。点击<code>Connect</code>，之后就可以可视化数据库中的数据。也可以使用鼠标简单的进行创建/删除数据库，新建/修改信息，也可以使用命令查找数据。</li>
+</ol></li>
+</ol>
+<h2 id="curd操作主要是查找数据">CURD操作（主要是查找数据）</h2>
+<p><strong>CRUD</strong></p>
+<ul>
+<li>C: Create Operations, 创建</li>
+<li>R: Read Operations, 读取</li>
+<li>U: Update Operations, 更新</li>
+<li>D: Delete Operations, 删除</li>
+</ul>
+<p>mongo数据库至上而下是：<code>Database</code>--&gt;<code>Collections</code>--&gt;<code>Documents</code>。我在这里分别理解成数据库、集合、记录。</p>
+<p>这里直接用官方给的数据来操作。假设以下数据放在了<code>fruits</code>集合中。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">-1</span><span class="punctuation">,</span> <span class="number">3</span><span class="punctuation">]</span><span class="punctuation">,</span> <span class="attr">&quot;links&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span><span class="attr">&quot;Uin&quot;</span><span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span> <span class="attr">&quot;NickName&quot;</span><span class="punctuation">:</span> <span class="string">&quot;123&quot;</span><span class="punctuation">&#125;</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">1</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;oranges&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="attr">&quot;in stock&quot;</span><span class="punctuation">:</span> <span class="number">8</span><span class="punctuation">,</span> <span class="attr">&quot;ordered&quot;</span><span class="punctuation">:</span> <span class="number">12</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span>  <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span> <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;avocados&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="string">&quot;fourteen&quot;</span> <span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h3 id="基本操作">基本操作</h3>
+<p><code>json</code>数据的嵌套查找，就是说，如果有字段包含json，可以直接用<code>.</code>进行连接。</p>
+<p>查找<code>qty</code>中的<code>ordered</code>为12的字段</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty.ordered&quot;: <span class="number">12</span>&#125;)</span><br></pre></td></tr></table></figure>
+<p>多重筛选，同时满足多个要求的记录</p>
+<p>查找<code>qty</code>为5，且<code>_id</code>为"apples"的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>, &quot;_id&quot;: &quot;apples&quot;&#125;)</span><br></pre></td></tr></table></figure>
+<p>第一个大括号内容表示筛选条件，第二个大括号表示显示内容</p>
+<p>筛选条件就不说明了，第二个大括号表示打印<code>_id</code>字段和<code>score</code>字段，不打印其他字段，</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123; &quot;_id&quot;: <span class="number">1</span> , &quot;score&quot;: <span class="number">1</span>&#125;)</span><br></pre></td></tr></table></figure>
+<h3 id="比较">比较</h3>
+<p>下面至上而下分别表示，在<code>fruits</code>集合中查找字段<code>qty</code>与4的大小关系所有记录。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">4</span>&#125;)          # 等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">4</span>&#125;&#125;)   # 大于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lt&quot;: <span class="number">4</span>&#125;&#125;)   # 小于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gte&quot;: <span class="number">4</span>&#125;&#125;)  # 大于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lte&quot;: <span class="number">4</span>&#125;&#125;)  # 小于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$ne&quot;: <span class="number">4</span>&#125;&#125;)   # 不等于</span><br></pre></td></tr></table></figure>
+<p>当然，也可以两个比较符一起操作，比如下面这样。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">3</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
+<p>如果对<code>score</code>这个字段使用比较，比如下面这样。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;score&quot;: &#123;&quot;$gt&quot;: <span class="number">2</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
+<p>将会返回下面两条记录。理由是，字段里的数组里有含有大于2小于5的数字。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">3</span></span><br><span class="line">    <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span></span><br><span class="line">            <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;123&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">/* 2 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">5</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h3 id="关系">关系</h3>
+<h4 id="in">$in</h4>
+<p>查找<code>_id</code>字段中包含<code>5</code>或者<code>ObjectId("507c35dd8fada716c89d0013")</code>的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$in&quot;:[<span class="number">5</span>, ObjectId(&quot;507c35dd8fada716c89d0013&quot;)] &#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="elemmatch">$elemMatch</h4>
+<p>用于字段中含有数组，且数组中含有<code>json</code>数据的的查找。</p>
+<p>查找<code>links</code>这个含有数组的字段，找出数组中<code>Uin</code>等于124的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;links&quot;: &#123;&quot;$elemMatch&quot;: &#123;&quot;Uin&quot;: <span class="number">124</span>&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="slice">$slice</h4>
+<p>切片，只打印字段中数组前n个数据</p>
+<p>筛选出记录之后，只打印记录<code>score</code>字段的前1条数据</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123;&quot;_id&quot;: <span class="number">1</span>, &quot;score&quot;: &#123;&quot;$slice&quot;: <span class="number">1</span>&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<p>输出如下</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h4 id="regex">$regex</h4>
+<p>使用正则表达式，更多操作请查看官方文档: <a
+href="https://docs.mongodb.com/manual/reference/operator/query/regex/index.html">$regex</a></p>
+<p>找出<code>_id</code>字段中所有以a开头的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$regex&quot;: &quot;^a&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h3 id="后缀">后缀</h3>
+<h4 id="sort排序">sort——排序</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">1</span>&#125;) # 升序</span><br><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">-1</span>&#125;) # 降序</span><br></pre></td></tr></table></figure>
+<h4 id="limit限制">limit——限制</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().limit(<span class="number">3</span>) # 对查询结果只输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
+<h4 id="skip跳过">skip——跳过</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">skip</span>(<span class="number">3</span>) # 对查询结果跳过输出前<span class="number">3</span>个，即不输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
+<h4 id="count计数">count——计数</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="built_in">count</span>()  # 返回查询结果总数</span><br></pre></td></tr></table></figure>
+<h4
+id="collation基于特定语言进行操作">collation——基于特定语言进行操作</h4>
+<p>没有怎么看懂，我理解的大概意思就是说，运行根据特定语言来进行查找。感觉用的不多，这里贴上官方给的例子</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">collation</span>( &#123; locale: &quot;en_US&quot;, strength: <span class="number">1</span> &#125; )</span><br></pre></td></tr></table></figure>
+<h3 id="个人常用操作">个人常用操作</h3>
+<p>例子： <figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">// test_db</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> ObjectId(<span class="string">&quot;56063f17ade2f21f36b03133&quot;</span>)<span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;title&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 是一个 Nosql 数据库&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;by&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;菜鸟教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;url&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;http://www.runoob.com&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;tags&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">           <span class="string">&quot;mongodb&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;database&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;NoSQL&quot;</span></span><br><span class="line">   <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@a1bf7a55ad8478017b95124774687769&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">12266535</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;一二三&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@5571e120b5d8f54f93338c76f62b9c419b818c29373dca9f331c2911a203bfbe&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> NumberLong(<span class="number">3379331828</span>)<span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;灵灵八&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure></p>
+<h4 id="返回查找结果的数目">返回查找结果的数目</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;title&quot;: &quot;MongoDB 教程&quot;&#125;).<span class="built_in">count</span>()</span><br></pre></td></tr></table></figure>
+<h4
+id="查找某个key中的数组是否某包含某内容">查找某个key中的数组是否某包含某内容</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;tags&quot;: &#123;&quot;$in&quot;: [&quot;mongodb&quot;]&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="使用正则进行查找">使用正则进行查找</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;url&quot;: &#123;&quot;$regex&quot;: &quot;^http&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="查找某个由数组构成的key方法">查找某个由数组构成的key方法</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;links&quot;:&#123;&quot;$elemMatch&quot;:&#123;&quot;Uin&quot;:<span class="number">12266535</span>,&quot;NickName&quot;:&quot;一二三&quot;&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h2 id="附录">附录</h2>
+<h3 id="数据库权限说明表">数据库权限说明表</h3>
+<table>
+<colgroup>
+<col style="width: 23%" />
+<col style="width: 76%" />
+</colgroup>
+<thead>
+<tr>
+<th>名称</th>
+<th style="text-align: left;">权限</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>数据库用户角色</td>
+<td style="text-align: left;">read、readWrite</td>
+</tr>
+<tr>
+<td>数据库管理角色</td>
+<td style="text-align: left;">dbAdmin、dbOwner、userAdmin</td>
+</tr>
+<tr>
+<td>集群管理角色</td>
+<td
+style="text-align: left;">clusterAdmin、clusterManager、clusterMonitor、hostManager</td>
+</tr>
+<tr>
+<td>备份恢复角色</td>
+<td style="text-align: left;">backup、restore</td>
+</tr>
+<tr>
+<td>所有数据库角色</td>
+<td
+style="text-align: left;">readAnyDatabase、readWriteAnyDatabase、userAdminAnyDatabase、dbAdminAnyDatabase</td>
+</tr>
+<tr>
+<td>超级用户角色</td>
+<td style="text-align: left;">root</td>
+</tr>
+<tr>
+<td>提供了系统超级用户的访问</td>
+<td style="text-align: left;">dbOwner
+、userAdmin、userAdminAnyDatabase</td>
+</tr>
+<tr>
+<td>内部角色</td>
+<td style="text-align: left;">__system</td>
+</tr>
+</tbody>
+</table>
+<p>下一篇：<a href="/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/" title="Mongodb：副本集配置">Mongodb：副本集配置</a></p>
 ]]></content>
       <categories>
-        <category>DeepLearning</category>
+        <category>Database</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>机器学习</tag>
-        <tag>神经网络</tag>
-        <tag>NumPy</tag>
+        <tag>mongodb</tag>
+        <tag>数据库</tag>
       </tags>
   </entry>
   <entry>
-    <title>神经网络笔记Ⅱ</title>
-    <url>/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
-    <content><![CDATA[<p>接 <a href="/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/" title="神经网络笔记Ⅰ">神经网络笔记Ⅰ</a></p>
-<p>从CNN到RNN再到LSTM，深入了解三种神经网络。其中，CNN主要用于图像识别与处理，RNN处理具有动态时序行为的任务，而LSTM则在此基础上添加了记忆单元，更适合处理时间序列中间隔和延迟非常长的事件。仅靠前向传播已不满足要求，反向传播也变得重要。</p>
-<span id="more"></span>
-<h2 id="cnn卷积神经网络">CNN（卷积神经网络）</h2>
+    <title>SQL语句的优化</title>
+    <url>/hexo_blog/2018/04/07/%E6%95%B0%E6%8D%AE%E5%BA%93/SQL%E8%AF%AD%E5%8F%A5%E7%9A%84%E4%BC%98%E5%8C%96/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了关于SQL语句（主要是Oracle数据库）的优化方案。
+<span id="more"></span></p>
 <p><a
-href="https://www.zybuluo.com/hanbingtao/note/485480">零基础入门深度学习(4)
-- 卷积神经网络</a></p>
-<p>CNN（Convolutional Neural
-Netword，卷积神经网络）是一种前馈神经网络，人工神经元可以响应周围单元，可以大型图像处理。卷积神经网络包括卷积层和池化层。</p>
-<p>基础结构</p>
-<ul>
-<li>特征提取层，每个神经元的输入与前一层的局部接受域相连，并提取该局部的特征。一旦该局部特征被提取后，它与其他特征间的位置关系也随之确定下来</li>
-<li>特征映射层，网络的每个计算层由多个特征映射组成，每个特征映射是一个平面，平面上所有的神经元的权值相等。采用影响函数较小的<span
-class="math inline">\(sigmoid\)</span>函数作为卷积网络的激活函数，使得特征映射具有位移不变性</li>
-</ul>
-<p>CNN主要用来识别位移、缩放以及其他形式扭曲不变性的二维图形。由于CNN的特征检测层通过训练数据进行学习，所以在使用CNN时，避免了显示的特征抽取，从而隐式地从训练数据中进行学习。由于同一特征映射面上的神经元权值相同，所以网络可以并行学习。</p>
-<p>对于图像识别任务</p>
-<p>全连接网络</p>
+href="https://juejin.im/post/5ab0f39df265da23866fba55">数据库性能优化之SQL语句优化</a></p>
+<h2 id="操作符优化">操作符优化</h2>
+<h3 id="in-操作符">IN 操作符</h3>
+<p>优点：编写容易，清晰易懂</p>
+<p>缺点：性能较低。</p>
+<p>Oracle执行步骤：</p>
 <ol type="1">
-<li>参数数量过多</li>
-<li>像素间的位置信息</li>
-<li>网络层数限制</li>
+<li>将其转换多表连接</li>
+<li>如果转换不成功则先执行IN里面的子查询，再执行外层表记录</li>
+<li>如果转换成功则直接采用多表连接查询</li>
+</ol>
+<p>所以用IN的SQL至少会多一个转换过程。</p>
+<p>替代方案：在业务密集的SQL用EXISTS替代</p>
+<h3 id="not-in操作符">NOT IN操作符</h3>
+<p>不能应用表的索引，不建议使用。</p>
+<p>替代方案：用NOT EXISTS方案</p>
+<h3 id="is-null或is-not-null操作">IS NULL或IS NOT NULL操作</h3>
+<p>判断字段为空一般是不会用索引的（索引不索引空值）。如果某列存在空值，即使对该列建索引也不会提高性能。</p>
+<p>替代方案：将<code>a is not null</code>改为<code>a&gt;0</code>或<code>a&gt;' '</code>等。不允许字段为空，而用一个缺省值代替空值。</p>
+<h3 id="及-操作符">&lt; 及 &gt; 操作符</h3>
+<p>一般情况下是不需要调整的，在某些情况下可以进行优化。</p>
+<p>如：一个表有100万记录，一个数值型字段A，有30万的A=0，39万A=2，
+1万A=3。在这种情况下执行A&gt;2与A&gt;=3有很大区别，<code>A&gt;2</code>在Oracle中会先找出为2的记录索引再进行比较，但是<code>A&gt;=3</code>会直接找到等于3的记录索引。</p>
+<h3 id="like操作符">LIKE操作符</h3>
+<p>LIKE医用通配符查询，基本上可以达到任一查询。但是不同语句可能会产生性能上的很大差异。如<code>LIKE '%5400%'</code>这种情况下不会引用索引，<code>LIKE 'X5400%'</code>会引用范围索引。即前者会产生全表扫描，后者会进行范围内的查询，后者相对于前者的性能大大提高了。</p>
+<h3 id="union操作符">UNION操作符</h3>
+<p>UNION在进行表连接之后会筛选掉重复的记录，所以在表连接之后会对产生的结果集进行排序运算，删除重复记录再返回结果。但是实际上大部分应用中是不会产生重复的记录，最常见的是过程表于历史表UNION。在这种情况下，替代方案就是使用UNION
+ALL代替UNION。UNION ALL操作只是简单的将两个结果合并后就返回</p>
+<h3 id="联接列">联接列</h3>
+<p>对于有联接的列，优化器是不会使用索引的。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 使用联接查询</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employss <span class="keyword">where</span> first_name<span class="operator">||</span><span class="string">&#x27;&#x27;</span><span class="operator">||</span>last_name <span class="operator">=</span><span class="string">&#x27;Beill Cliton&#x27;</span>;</span><br><span class="line"># 不使用联接查询，基于last_name创建索引</span><br><span class="line"><span class="keyword">where</span> first_name <span class="operator">=</span><span class="string">&#x27;Beill&#x27;</span> <span class="keyword">and</span> last_name <span class="operator">=</span><span class="string">&#x27;Cliton&#x27;</span>;</span><br></pre></td></tr></table></figure>
+<h3 id="order-by语句">Order By语句</h3>
+<p>将返回的查询结果进行排序。任何在Order
+by语句的非索引项或者有计算表达式都将降低查询速度。解决方案就是为所使用的列建立另一个索引，同时也绝对避免在order
+by子句中使用表达式。</p>
+<h3 id="not-语句">NOT 语句</h3>
+<p>NOT可用来对任何逻辑运算符号取反。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 如果要使用<span class="keyword">NOT</span>，则应在取反的短语前面机上括号，并在短语前面机上<span class="keyword">NOT</span>运算符。</span><br><span class="line"><span class="keyword">where</span> <span class="keyword">not</span> (status <span class="operator">=</span> <span class="string">&#x27;VALID&#x27;</span>)</span><br><span class="line"># <span class="keyword">NOT</span>运算符包含在另外一个逻辑运算符<span class="operator">&lt;&gt;</span>运算符中。即使不在查询的<span class="keyword">where</span>子句中显式地加入<span class="keyword">NOT</span>词，<span class="keyword">NOT</span>仍在运算符中</span><br><span class="line"><span class="keyword">where</span> status <span class="operator">&lt;&gt;</span> <span class="string">&#x27;INVALID&#x27;</span>;</span><br><span class="line"># 这种情况运行建立salary的索引，在速度上会优于上面的方案</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employee <span class="keyword">where</span> salary <span class="operator">&lt;</span> <span class="number">3000</span> <span class="keyword">or</span> salary <span class="operator">&gt;</span> <span class="number">3000</span>;</span><br></pre></td></tr></table></figure>
+<h2 id="书写的影响">书写的影响</h2>
+<h3
+id="同一功能同一性能不同写法sql的影响">同一功能同一性能不同写法SQL的影响</h3>
+<p>尽可能避免不同写法的出现，根据Oracle共享内存SGA原理，如果语句不同则进行分析，占用共享内存。如果SQL字符串及格式完全相同，则Oracle只会分析一次，共享内存也只会留下一次的分析结果</p>
+<h3 id="where后面的条件顺序影响">WHERE后面的条件顺序影响</h3>
+<p>下面两个语句，在没有建立索引的情况下，执行都是全表扫描。</p>
+<p><code>dy_dj = '1KV以下'</code>这个条件在记录集内比率为99%,
+<code>xh_bz=1</code>而这个条件只有0.5%。所以第二条SQL的CPU占用率明显低于第一条。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span> <span class="keyword">and</span> xh_bz<span class="operator">=</span><span class="number">1</span></span><br><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> xh_bz<span class="operator">=</span><span class="number">1</span> <span class="keyword">and</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="查询表顺序的影响">查询表顺序的影响</h3>
+<p>在FROM后面的表中的列表顺序会对SQL执行性能影响，在没有索引及ORACLE没有对表进行统计分析的情况下，ORACLE会按<strong>表出现的顺序</strong>进行连接，所以如果表的顺序不对时会产生十分消耗服务器资源的数据交叉（如果对表进行了统计分析，Oracle自动先进小表的连接，再进行大表的连接）</p>
+<h2 id="sql语句索引的利用">SQL语句索引的利用</h2>
+<h3 id="条件字段的优化">条件字段的优化</h3>
+<p>函数处理的字段不能利用索引</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：substr(hbs_bh,<span class="number">1</span>,<span class="number">4</span>)<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span></span><br><span class="line">优化处理：hbs_bh <span class="keyword">like</span> <span class="string">&#x27;5400%&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：trunc(sk_rq)<span class="operator">=</span>trunc(sysdate)</span><br><span class="line">优化处理：sk_rq<span class="operator">&gt;=</span>trunc(sysdate) <span class="keyword">and</span> sk_rq<span class="operator">&lt;</span>trunc(sysdate<span class="operator">+</span><span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>进行了显示或隐式的运算的字段不能进行索引。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：<span class="string">&#x27;X&#x27;</span> <span class="operator">||</span> hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;X5400021452&#x27;</span></span><br><span class="line">优化处理：hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;5400021542&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：sk_rq<span class="operator">+</span><span class="number">5</span><span class="operator">=</span>sysdate</span><br><span class="line">优化处理：sk_rq<span class="operator">=</span>sysdate<span class="number">-5</span></span><br></pre></td></tr></table></figure>
+<p>条件内包括的多个本表的字段运算时不能进行索引</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：ys_df<span class="operator">&gt;</span>cx_df</span><br><span class="line">无法进行优化</span><br><span class="line"></span><br><span class="line">原语句：qc_bh <span class="operator">||</span> kh_bh<span class="operator">=</span><span class="string">&#x27;5400250000&#x27;</span></span><br><span class="line">优化处理：qc_bh<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span> <span class="keyword">and</span> kh_bh<span class="operator">=</span><span class="string">&#x27;250000&#x27;</span></span><br></pre></td></tr></table></figure>
+<h2 id="其他优化方法">其他优化方法</h2>
+<ol type="1">
+<li><p>选择最有效率的表名顺序（只在基于规则的优化器中有效）</p>
+<p><code>ORACLE</code>
+的解析器按照从右到左的顺序处理FROM子句中的表名，FROM子句中写在最后的表(基础表
+driving
+table)将被最先处理，在FROM子句中包含多个表的情况下,你必须选择记录条数最少的表作为基础表。如果有3个以上的表连接查询,
+那就需要选择交叉表(intersection table)作为基础表,
+交叉表是指那个被其他表所引用的表</p></li>
+<li><p><code>WHERE</code>子句中的连接顺序</p>
+<p><code>ORACLE</code>采用自下而上的顺序解析WHERE子句,根据这个原理,表之间的连接必须写在其他<code>WHERE</code>条件之前,
+那些可以过滤掉最大数量记录的条件必须写在WHERE子句的末尾.</p></li>
+<li><p><code>SELECT</code>子句中避免使用 '*'</p>
+<p><code>ORACLE</code>在解析的过程中, 会将'*'依次转换成所有的列名,
+这个工作是通过查询数据字典完成的, 这意味着将耗费更多的时间。</p></li>
+<li><p>减少访问数据库的次数</p>
+<p><code>ORACLE</code>在内部执行了许多工作: 解析SQL语句,
+估算索引的利用率, 绑定变量 , 读数据块等。</p></li>
+<li><p>在<code>SQL*Plus</code> ,
+<code>SQL*Forms</code>和<code>Pro*C</code>中重新设置<code>ARRAYSIZE</code>参数,
+可以增加每次数据库访问的检索数据量 ,建议值为200</p></li>
+<li><p>使用<code>DECODE</code>函数来减少处理时间</p>
+<p>使用<code>DECODE</code>函数可以避免重复扫描相同记录或重复连接相同的表.</p></li>
+<li><p>整合简单,无关联的数据库访问</p></li>
+</ol>
+<p>如果你有几个简单的数据库查询语句,你可以把它们整合到一个查询中(即使它们之间没有关系)
+。</p>
+<ol start="8" type="1">
+<li>删除重复记录：</li>
+</ol>
+<blockquote>
+<p>最高效的删除重复记录方法(因为使用了<code>ROWID</code>)</p>
+</blockquote>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">DELETE</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.ROWID <span class="operator">&gt;</span> (<span class="keyword">SELECT</span> <span class="built_in">MIN</span>(X.ROWID) <span class="keyword">FROM</span> EMP X <span class="keyword">WHERE</span> X.EMP_NO <span class="operator">=</span> E.EMP_NO)</span><br></pre></td></tr></table></figure>
+<ol start="9" type="1">
+<li><p>用<code>TRUNCATE</code>替代<code>DELETE</code></p>
+<p>当删除表中的记录时,在通常情况下, 回滚段(rollback segments )
+用来存放可以被恢复的信息.
+如果你没有<code>COMMIT</code>事务,<code>ORACLE</code>会将数据恢复到删除之前的状态(准确地说是恢复到执行删除命令之前的状况)
+而当运用<code>TRUNCATE</code>时,
+回滚段不再存放任何可被恢复的信息.当命令运行后,数据不能被恢复.因此很少的资源被调用,执行时间也会很短.
+(译者按:
+<code>TRUNCATE</code>只在删除全表适用,<code>TRUNCATE</code>是DDL不是DML)
+。</p></li>
+<li><p>尽量多使用<code>COMMIT</code></p>
+<p>只要有可能,在程序中尽量多使用<code>COMMIT</code>,
+这样程序的性能得到提高,需求也会因为<code>COMMIT</code>所释放的资源而减少，<code>COMMIT</code>所释放的资源:</p>
+<ol type="a">
+<li><p>回滚段上用于恢复数据的信息.</p></li>
+<li><p>被程序语句获得的锁</p></li>
+<li><p>redo log buffer 中的空间</p></li>
+<li><p><code>ORACLE</code>为管理上述3种资源中的内部花费</p></li>
+</ol></li>
+<li><p>用<code>Where</code>子句替换<code>HAVING</code>子句</p></li>
+</ol>
+<pre><code>避免使用`HAVING`子句, `HAVING` 只会在检索出所有记录之后才对结果集进行过滤. 这个处理需要排序,总计等操作. 如果能通过`WHERE`子句限制记录的数目,那就能减少这方面的开销. (非`oracle`中)`on`、`where`、`having`这三个都可以加条件的子句中，`on`是最先执行，`where`次之，`having`最后，因为`on`是先把不符合条件的记录过滤后才进行统计，它就可以减少中间运算要处理的数据，按理说应该速度是最快的，`where`也应该比`having`快点的，因为它过滤数据后才进行`sum`，在两个表联接时才用`on`的，所以在一个表的时候，就剩下`where`跟`having`比较了。在这单表查询统计的情况下，如果要过滤的条件没有涉及到要计算字段，那它们的结果是一样的，只是`where`可以使用`rushmore`技术，而`having`就不能，在速度上后者要慢如果要涉及到计算的字 段，就表示在没计算之前，这个字段的值是不确定的，根据上篇写的工作流程，`where`的作用时间是在计算之前就完成的，而`having`就是在计算后才起作 用的，所以在这种情况下，两者的结果会不同。在多表联接查询时，`on`比`where`更早起作用。系统首先根据各个表之间的联接条件，把多个表合成一个临时表 后，再由`where`进行过滤，然后再计算，计算完后再由`having`进行过滤。由此可见，要想过滤条件起到正确的作用，首先要明白这个条件应该在什么时候起作用，然后再决定放在那里。</code></pre>
+<ol start="12" type="1">
+<li>减少对表的查询</li>
+</ol>
+<blockquote>
+<p>在含有子查询的SQL语句中,要特别注意减少对表的查询</p>
+</blockquote>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> TAB_NAME <span class="keyword">FROM</span> TABLES <span class="keyword">WHERE</span> (TAB_NAME,DB_VER) <span class="operator">=</span> ( <span class="keyword">SELECT</span> TAB_NAME,DB_VER <span class="keyword">FROM</span> TAB_COLUMNS <span class="keyword">WHERE</span> VERSION <span class="operator">=</span> <span class="number">604</span>)</span><br></pre></td></tr></table></figure>
+<ol start="13" type="1">
+<li><p>通过内部函数提高SQL效率</p>
+<p>复杂的SQL往往牺牲了执行效率.
+能够掌握上面的运用函数解决问题的方法在实际工作中是非常有意义的。</p></li>
+<li><p>使用表的别名(<code>Alias</code>)：</p></li>
+</ol>
+<pre><code>当在SQL语句中连接多个表时, 请使用表的别名并把别名前缀于每个Column上.这样一来,就可以减少解析的时间并减少那些由Column歧义引起的语法错误。</code></pre>
+<ol start="15" type="1">
+<li>识别’低效执行’的SQL语句</li>
+</ol>
+<pre><code>虽然目前各种关于SQL优化的图形化工具层出不穷,但是写出自己的SQL工具来解决问题始终是一个最好的方法。</code></pre>
+<p>​<code>sql SELECT EXECUTIONS, DISK_READS, BUFFER_GETS, ROUND((BUFFER_GETS-DISK_READS)/BUFFER_GETS,2) Hit_radio, ROUND(DISK_READS/EXECUTIONS,2) Reads_per_run, SQL_TEXT FROM V$SQLAREA WHERE EXECUTIONS&gt;0 AND BUFFER_GETS &gt; 0 AND (BUFFER_GETS-DISK_READS)/BUFFER_GETS &lt; 0.8 ORDER BY 4 DESC; ​</code></p>
+<ol start="16" type="1">
+<li><p>用索引提高效率</p>
+<p>索引是表的一个概念部分,用来提高检索数据的效率，ORACLE使用了一个复杂的自平衡B-tree结构.
+通常,通过索引查询数据比全表扫描要快.
+当ORACLE找出执行查询和Update语句的最佳路径时, ORACLE优化器将使用索引.
+同样在联结多个表时使用索引也可以提高效率.
+另一个使用索引的好处是,它提供了主键(primary
+key)的唯一性验证.。那些LONG或LONG RAW数据类型, 你可以索引几乎所有的列.
+通常, 在大型表中使用索引特别有效. 当然,你也会发现,
+在扫描小表时,使用索引同样能提高效率.
+虽然使用索引能得到查询效率的提高,但是我们也必须注意到它的代价.
+索引需要空间来存储,也需要定期维护, 每当有记录在表中增减或索引列被修改时,
+索引本身也会被修改. 这意味着每条记录的INSERT , DELETE ,
+UPDATE将为此多付出4 , 5 次的磁盘I/O .
+因为索引需要额外的存储空间和处理,那些不必要的索引反而会使查询反应时间变慢.。定期的重构索引是有必要的</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">ALTER</span> INDEX <span class="operator">&lt;</span>INDEXNAME<span class="operator">&gt;</span> REBUILD <span class="operator">&lt;</span>TABLESPACENAME<span class="operator">&gt;</span></span><br></pre></td></tr></table></figure>
+<ol start="17" type="1">
+<li><p>用<code>EXISTS</code>替换<code>DISTINCT</code></p>
+<p>当提交一个包含一对多表信息(比如部门表和雇员表)的查询时,避免在<code>SELECT</code>子句中使用<code>DISTINCT</code>.
+一般可以考虑用<code>EXIST</code>替换, <code>EXISTS</code>
+使查询更为迅速,因为RDBMS核心模块将在子查询的条件一旦满足后,立刻返回结果.
+例子：</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">(低效):</span><br><span class="line"><span class="keyword">SELECT</span> <span class="keyword">DISTINCT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D, EMP E <span class="keyword">WHERE</span> D.DEPT_NO <span class="operator">=</span> E.DEPT_NO</span><br><span class="line">(高效):</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D <span class="keyword">WHERE</span> <span class="keyword">EXISTS</span> ( <span class="keyword">SELECT</span> <span class="string">&#x27;X&#x27;</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.DEPT_NO <span class="operator">=</span> D.DEPT_NO);</span><br></pre></td></tr></table></figure>
+<ol start="18" type="1">
+<li><p>sql语句用大写的</p>
+<p>因为oracle总是先解析sql语句，把小写的字母转换成大写的再执行</p></li>
+<li><p>在java代码中尽量少用连接符“＋”连接字符串！</p></li>
+<li><p>避免在索引列上使用NOT，通常我们要避免在索引列上使用NOT,
+NOT会产生在和在索引列上使用函数相同的影响.
+当ORACLE”遇到”NOT,他就会停止使用索引转而执行全表扫描。</p></li>
+<li><p>避免在索引列上使用计算</p>
+<p>WHERE子句中，如果索引列是函数的一部分．优化器将不使用索引而使用全表扫描．</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">*</span> <span class="number">12</span> <span class="operator">&gt;</span> <span class="number">25000</span>;</span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">&gt;</span> <span class="number">25000</span><span class="operator">/</span><span class="number">12</span>;</span><br></pre></td></tr></table></figure>
+<ol start="22" type="1">
+<li>用&gt;=替代&gt;</li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;=</span> <span class="number">4</span></span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;</span> <span class="number">3</span></span><br></pre></td></tr></table></figure>
+<ol start="23" type="1">
+<li><p>用UNION替换OR (适用于索引列)</p>
+<p>通常情况下, 用UNION替换WHERE子句中的OR将会起到较好的效果.
+对索引列使用OR将造成全表扫描. 注意, 以上规则只针对多个索引列有效.
+如果有column没有被索引, 查询效率可能会因为你没有选择OR而降低.
+在下面的例子中, LOC_ID 和REGION上都建有索引.</p></li>
 </ol>
-<p>解决方案：</p>
-<ol type="1">
-<li>局部连接——不是全连接，减少参数数量</li>
-<li>权值共享——一组连接可以共享共一个权重，再一次减少参数数量</li>
-<li>下采样——使用池化（Pooling）来减少每层的样本数，进一步减少参数数量，提供<strong>鲁棒性
-</strong></li>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br></pre></td></tr></table></figure>
+<ol start="24" type="1">
+<li><p>用IN来替换OR</p>
+<p>这是一条简单易记的规则，但是实际的执行效果还须检验，在ORACLE8下，两者的执行路径似乎是相同的．</p></li>
 </ol>
-<p><strong>P.S.</strong></p>
-<p>鲁棒是Robust的音译，也就是健壮和强壮的意思。它是在异常和危险情况下系统生存的关键。比如说，计算机软件在输入错误、磁盘故障、网络过载或有意攻击情况下，能否不死机、不崩溃，就是该软件的鲁棒性。所谓“鲁棒性”，是指控制系统在一定（结构，大小）的参数摄动下，维持其它某些性能的特性。根据对性能的不同定义，可分为稳定鲁棒性和性能鲁棒性。以闭环系统的鲁棒性作为目标设计得到的固定控制器称为鲁棒控制器。</p>
-<p>卷积神经网络：<strong>卷积层、Pooling层（池化层）、全连接层</strong></p>
-<p><code>INPUT -&gt; [[CONV]*N -&gt; POOL]*M -&gt; [FC]*K</code></p>
-<p>N个卷积层叠加再加上一个池化层，重复M次，加上K个全连接层。</p>
-<p>概念介绍和计算方式请看下面这篇博客</p>
-<p><a
-href="http://www.cnblogs.com/charlotte77/p/7759802.html">【深度学习系列】卷积神经网络CNN原理详解(一)——基本原理</a></p>
-<p>总结：</p>
-<ol type="1">
-<li>池化方式：
-<ol type="1">
-<li>MaxPooling：取滑动窗口里最大的值</li>
-<li>AveragePooling：取滑动窗口内所有值的平均值</li>
-<li>StochasticPooling，介于两者之间，通过对像素点按照数值大小赋予概率，再按照概率进行亚采样；</li>
-</ol></li>
-<li>Zero Padding(补零)：保留原始图片的尺寸</li>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span>…. <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">20</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">30</span></span><br><span class="line">高效</span><br><span class="line"><span class="keyword">SELECT</span>… <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_IN <span class="keyword">IN</span> (<span class="number">10</span>,<span class="number">20</span>,<span class="number">30</span>);</span><br></pre></td></tr></table></figure>
+<ol start="25" type="1">
+<li><p>避免在索引列上使用IS NULL和IS NOT NULL</p>
+<p>避免在索引中使用任何可以为空的列，ORACLE将无法使用该索引．对于单列索引，如果列包含空值，索引中将不存在此记录.
+对于复合索引，如果每个列都为空，索引中同样不存在此记录.
+如果至少有一个列不为空，则记录存在于索引中．举例:
+如果唯一性索引建立在表的A列和B列上,
+并且表中存在一条记录的A,B值为(123,null) ,
+ORACLE将不接受下一条具有相同A,B值（123,null）的记录(插入).
+然而如果所有的索引列都为空，ORACLE将认为整个键值为空而空不等于空.
+因此你可以插入1000 条具有相同键值的记录,当然它们都是空!
+因为空值不存在于索引列中,所以WHERE子句中对索引列进行空值比较将使ORACLE停用该索引.</p></li>
 </ol>
-<h4 id="卷积">卷积</h4>
-<p>对于一个<span class="math inline">\(l_1*l_2\)</span>的图像，使用<span
-class="math inline">\(f_1*f_2\)</span>的filter进行卷积（后者小于前者），得到一个<span
-class="math inline">\(f_3*f_4\)</span>的feature map。</p>
-<p><span class="math display">\[
-a_{ij}=f(\sum\limits_{m=0}^{f_1-1}\sum\limits_{n=0}^{f_2-1}w_{mn}x_{(m+i)(n+j)}+w_b)
-\]</span></p>
-<p><span class="math inline">\(a_{ij}\)</span>表示的是feature
-map的元素，<span
-class="math inline">\(w_{mn}\)</span>表示对应元素，<span
-class="math inline">\(f\)</span>函数表示激活函数，<span
-class="math inline">\(x_{(m+i)(n+j)}\)</span>表示图像的元素，<span
-class="math inline">\(w_b\)</span>表示偏置项，这里从0开始计数，所以需要-1。</p>
-<p>这里简单起见，步幅设置为1。</p>
-<p>图像大小、步幅和卷积后的Feature Map大小是有关系的。</p>
-<p><span class="math inline">\(W_2=(W_1-F+2P)/S+1\)</span></p>
-<p><span class="math inline">\(H_2=(H_1-F+2P)/S+1\)</span></p>
-<p><span class="math inline">\(W_2\)</span>是卷积后Feature
-Map的宽度；<span
-class="math inline">\(W_1\)</span>是卷积前图像的宽度；<span
-class="math inline">\(F\)</span>是filter的宽度；<span
-class="math inline">\(P\)</span>是<strong>Zero
-Padding</strong>数量，<strong>Zero
-Padding</strong>是指在原始图像周围补几圈0，如果的值是1，那么就补1圈0；<span
-class="math inline">\(S\)</span>是<strong>步幅</strong>；<span
-class="math inline">\(H_2\)</span>是卷积后Feature Map的高度；<span
-class="math inline">\(H_1\)</span>是卷积前图像的宽度。</p>
-<p>二维卷积公式，已知矩阵<span class="math inline">\(A\)</span>、<span
-class="math inline">\(B\)</span>，行列数分别用<span
-class="math inline">\(m\)</span>和<span
-class="math inline">\(n\)</span>表示。卷积公式如下</p>
-<p><span
-class="math inline">\(C_{st}=\sum\limits_0^{m_A-1}\sum\limits_0^{n_A-1}A_{mn}B_{(s-m)(t-n)}\)</span></p>
-<p><span class="math inline">\(0\le s&lt;m_A+m_B-1\)</span></p>
-<p><span class="math inline">\(0\le t&lt;n_A+n_B-1\)</span></p>
-<p>也可以写成<span class="math inline">\(C=A*B\)</span></p>
-<p>数学中的卷积与卷积神经网络中的卷积是一定区别的。可以把神经网络中的卷积称为<strong>互相关(cross-correlation)</strong>。这两种操作转换方式，可以通过把矩阵<span
-class="math inline">\(A\)</span>翻转180度，再交换<span
-class="math inline">\(A\)</span>、<span
-class="math inline">\(B\)</span>运算位置，即可以将卷积变为互相关。</p>
-<h2 id="rnn循环神经网络">RNN（循环神经网络）</h2>
-<p>RNN（Recurrent neural
-Network，循环神经网络/多层反馈网络）是一种节点定向连接成环的人工神经网络。这种网络的内部状态可以展示动态时序行为。不同于前馈神经网络的是，RNN可以利用它内部的记忆来处理任意时序的输入序列，这让它可以更容易处理，如不分段的手写识别、语音识别等。</p>
-<p>RNN网络的本质特征是处理单元之间既有内部的反馈连接又有前馈连接。从系统的观点看，它是一个反馈动力系统，在计算过程中体现过程动态特性，比前馈神经网络具有更强的动态行为和计算能力。</p>
-<p><a
-href="https://iamtrask.github.io/2015/11/15/anyone-can-code-lstm/">参考文章</a></p>
-<p><strong><em>Talk</em> <em>is</em> cheap. <em>Show</em> <em>me</em>
-<em>the</em> <em>code</em>.</strong></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># sigmoid激活函数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储10进制转2进制的字典</span></span><br><span class="line">int2binary = &#123;&#125;</span><br><span class="line"><span class="comment"># 二进制数最大长度</span></span><br><span class="line">binary_dim = <span class="number">8</span></span><br><span class="line"><span class="comment"># 最大10进制数</span></span><br><span class="line">larget_number = <span class="number">2</span> ** binary_dim</span><br><span class="line">binary = np.unpackbits(</span><br><span class="line">    np.array([<span class="built_in">range</span>(larget_number)], dtype=np.uint8).T, axis=<span class="number">1</span>)</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(larget_number):</span><br><span class="line">    int2binary[i] = binary[i]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 梯度</span></span><br><span class="line">alpha = <span class="number">0.1</span></span><br><span class="line"><span class="comment"># 输入维度，两个数</span></span><br><span class="line">input_dim = <span class="number">2</span></span><br><span class="line"><span class="comment"># 隐单元维度</span></span><br><span class="line">hidden_dim = <span class="number">16</span></span><br><span class="line"><span class="comment"># 输出维度，一个数</span></span><br><span class="line">output_dim = <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 两层神经网络。每层权重，[-1, 1]</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((input_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, output_dim)) - <span class="number">1</span></span><br><span class="line"><span class="comment"># 神经网络连接前一个隐藏层的权重</span></span><br><span class="line">syn_h = <span class="number">2</span> * np.random.random((hidden_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储权重的更新</span></span><br><span class="line">syn_0_update = np.zeros_like(syn_0)</span><br><span class="line">syn_1_update = np.zeros_like(syn_1)</span><br><span class="line">syn_h_update = np.zeros_like(syn_h)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 一个简单的加法问题，c=a+b</span></span><br><span class="line">    <span class="comment"># 在0，128（避免相加超过256）之间一个随机数a，十进制表示</span></span><br><span class="line">    a_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    <span class="comment"># 二进制表示a</span></span><br><span class="line">    a = int2binary[a_int]</span><br><span class="line"></span><br><span class="line">    b_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    b = int2binary[b_int]</span><br><span class="line">    <span class="comment"># 正确答案c</span></span><br><span class="line">    c_int = a_int + b_int</span><br><span class="line">    c = int2binary[c_int]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 神经网络计算的答案</span></span><br><span class="line">    d = np.zeros_like(c)</span><br><span class="line">    <span class="comment"># 计算的误差</span></span><br><span class="line">    overallError = <span class="number">0</span></span><br><span class="line">    <span class="comment"># 记录l2_delta、l1</span></span><br><span class="line">    layer_2_deltas = []</span><br><span class="line">    layer_1_values = []</span><br><span class="line">    <span class="comment"># l1初始值全为0</span></span><br><span class="line">    layer_1_values.append(np.zeros(hidden_dim))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 前向传播。特征是二进制数维度。</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># X是1*2的矩阵，[[m, n]]，m、n是二进制数a、b在每个位置的01值。从最后一维开始遍历</span></span><br><span class="line">        X = np.array(</span><br><span class="line">            [[a[binary_dim - position - <span class="number">1</span>], b[binary_dim - position - <span class="number">1</span>]]])</span><br><span class="line">        <span class="comment"># y是1*1的矩阵。[[t]]，t是二进制数c在每个位置的01值</span></span><br><span class="line">        y = np.array([[c[binary_dim - position - <span class="number">1</span>]]]).T</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 计算输出层，np.dot(X,syn_0)是正常操作:从输入层到隐藏层;np.dot(layer_1_values[-1],syn_h):从前一层隐藏层到这一层隐藏层</span></span><br><span class="line">        layer_1 = sigmoid(np.dot(X, syn_0) + np.dot(layer_1_values[-<span class="number">1</span>], syn_h))</span><br><span class="line">        layer_2 = sigmoid(np.dot(layer_1, syn_1))</span><br><span class="line">        <span class="comment"># 计算l2偏差</span></span><br><span class="line">        layer_2_error = y - layer_2</span><br><span class="line">        <span class="comment"># 保存偏差加权导数l2_delta</span></span><br><span class="line">        layer_2_deltas.append(layer_2_error * sigmoid(layer_2, deriv=<span class="literal">True</span>))</span><br><span class="line">        <span class="comment"># 误差总和</span></span><br><span class="line">        overallError += np.<span class="built_in">abs</span>(layer_2_error[<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 神经网络每个维度的计算的01值</span></span><br><span class="line">        d[binary_dim - position - <span class="number">1</span>] = np.<span class="built_in">round</span>(layer_2[<span class="number">0</span>][<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 保存l1层，在下次迭代时计算下次传播的隐藏层</span></span><br><span class="line">        <span class="comment"># layer_1_values.append(copy.deepcopy(layer_1))</span></span><br><span class="line">        layer_1_values.append(layer_1.copy())</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 存储反向传播的l1_delta</span></span><br><span class="line">    futuer_layer_1_delta = np.zeros(hidden_dim)</span><br><span class="line">    <span class="comment"># 反向传播.(input + prev_input) -&gt; hidden -&gt; output</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># 从第一维开始遍历</span></span><br><span class="line">        X = np.array([[a[position], b[position]]])</span><br><span class="line">        <span class="comment"># 选出前面计算得到的l1作为当前层，反向计算</span></span><br><span class="line">        layer_1 = layer_1_values[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 选择l1前一层的隐藏层</span></span><br><span class="line">        prev_layer_1 = layer_1_values[-position - <span class="number">2</span>]</span><br><span class="line">        <span class="comment"># 选择出l2，当前层输出的误差</span></span><br><span class="line">        layer_2_delta = layer_2_deltas[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 计算当前隐藏层误差，计算前一层的l1_delta到当前层，加上正常操作。</span></span><br><span class="line">        layer_1_delta = (np.dot(futuer_layer_1_delta, syn_h.T) +</span><br><span class="line">                         layer_2_delta.dot(syn_1.T)) * sigmoid(layer_1, deriv=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 更新权重更新的参数。将l1转换为二维矩阵</span></span><br><span class="line">        syn_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)</span><br><span class="line">        syn_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># X已经是二维矩阵，不需要转换</span></span><br><span class="line">        syn_0_update += X.T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># 替换futuer_l1</span></span><br><span class="line">        futuer_layer_1_delta = layer_1_delta</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 调整每层权重</span></span><br><span class="line">    syn_0 += syn_0_update * alpha</span><br><span class="line">    syn_1 += syn_1_update * alpha</span><br><span class="line">    syn_h += syn_h_update * alpha</span><br><span class="line">    <span class="comment"># 归零更新权重参数</span></span><br><span class="line">    syn_0_update *= <span class="number">0</span></span><br><span class="line">    syn_1_update *= <span class="number">0</span></span><br><span class="line">    syn_h_update *= <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (_ % <span class="number">1000</span> == <span class="number">0</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Error: &quot;</span> + <span class="built_in">str</span>(overallError))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Pred: &quot;</span> + <span class="built_in">str</span>(d))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;True: &quot;</span> + <span class="built_in">str</span>(c))</span><br><span class="line">        <span class="comment"># 二进制转十进制</span></span><br><span class="line">        out = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> index, x <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">reversed</span>(d)):</span><br><span class="line">            out += x * (<span class="number">2</span>**index)</span><br><span class="line">        <span class="built_in">print</span>(<span class="built_in">str</span>(a_int) + <span class="string">&quot; + &quot;</span> + <span class="built_in">str</span>(b_int) + <span class="string">&quot;=&quot;</span> + <span class="built_in">str</span>(out))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;-------------&quot;</span>)</span><br></pre></td></tr></table></figure>
-<p>将前向传播比喻为字母表顺序读出，反向传播就是字母表反序读出。</p>
-<h2 id="lstm时间递归神经网络">LSTM（时间递归神经网络）</h2>
-<p>长短期记忆（英语：Long Short-Term
-Memory，LSTM）是一种时间递归神经网络（RNN），论文首次发表于1997年。由于独特的设计结构，LSTM适合于处理和预测时间序列中间隔和延迟非常长的重要事件。</p>
-<p>LSTM的表现通常比时间递归神经网络及隐马尔科夫模型（HMM）更好，比如用在不分段连续手写识别上。作为非线性模型，LSTM可作为复杂的非线性单元用于构造更大型深度神经网络。</p>
-]]></content>
-      <categories>
-        <category>DeepLearning</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>神经网络</tag>
-        <tag>NumPy</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Hadoop伪分布式安装（Centos7）</title>
-    <url>/hexo_blog/2017/04/09/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Hadoop%E4%BC%AA%E5%88%86%E5%B8%83%E5%BC%8F%E5%AE%89%E8%A3%85%EF%BC%88Centos7%EF%BC%89/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Hadoop的伪分布式安装和一些基础操作</p>
-<span id="more"></span>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/wKiom1SFSNnAeaEUAADD3ZcjTjw828.jpg" /></p>
-<h2 id="文章开头">文章开头</h2>
-<p>环境配置：Centos7</p>
-<p>Hadoop版本：hadoop：2.7.3</p>
-<p>JDK版本：jdk-8u111-linux-x64.tar.gz</p>
-<h2 id="安装之前">安装之前</h2>
-<ol type="1">
-<li><p>下载hadoop安装包, <a
-href="http://hadoop.apache.org/releases.html">下载网址</a>。进入官网之后，点击对应版本的“binary”，之后点击链接下载即可。</p></li>
-<li><p>jdk。通过官网下载jdk</p></li>
-<li><p>通过Xshell，上传两个安装包至/usr/local/。服务器与本地下载和上传需要lrzsz这个软件</p></li>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引失效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="keyword">IS</span> <span class="keyword">NOT</span> <span class="keyword">NULL</span>;</span><br><span class="line">高效: (索引有效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="operator">&gt;=</span><span class="number">0</span>;</span><br></pre></td></tr></table></figure>
+<ol start="26" type="1">
+<li><p>总是使用索引的第一个列</p>
+<p>如果索引是建立在多个列上, 只有在它的第一个列(leading
+column)被where子句引用时,优化器才会选择使用该索引.
+这也是一条简单而重要的规则，当仅引用索引的第二个列时,优化器使用了全表扫描而忽略了索引。</p></li>
+<li><p>用UNION-ALL 替换UNION ( 如果有可能的话)</p>
+<p>当SQL
+语句需要UNION两个查询结果集合时,这两个结果集合会以UNION-ALL的方式被合并,
+然后在输出最终结果前进行排序. 如果用UNION ALL替代UNION,
+这样排序就不是必要了. 效率就会因此得到提高. 需要注意的是，UNION ALL
+将重复输出两个结果集合中相同记录. 因此各位还是要从业务需求分析使用UNION
+ALL的可行性. UNION
+将对结果集合排序,这个操作会使用到SORT_AREA_SIZE这块内存.
+对于这块内存的优化也是相当重要的.
+下面的SQL可以用来查询排序的消耗量</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">yum install lrzsz</span><br></pre></td></tr></table></figure>
-<p>之后可手动拖动文件至服务器当前目录下，也可通过命令
-“rz”来打开窗口，从而进行上传。 （下载命令“sz”）</p>
-<h2 id="解压缩及配置">解压缩及配置</h2>
-<p>在<code>/usr/local/</code>目录下进行解压 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#请根据下载版本，进行解压缩</span></span><br><span class="line">tar -zxvf jdk-8u111-linux-x64.tar.gz</span><br><span class="line">tar -zxvf hadoop-2.7.3.tar.gz</span><br></pre></td></tr></table></figure></p>
-<h3
-id="配置java环境变量可与配置hadoop环境变量配置一起进行">配置Java环境变量（可与配置Hadoop环境变量配置一起进行）</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#第十行左右输入命令，注意版本</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
-<h3 id="配置hadoop环境变量">配置Hadoop环境变量</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#在刚刚JAVA_HOME之下一行，注意版本</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#此处的PATH承接上面Java环境变量的配置，即是在原来基础上进行增加而不是另起一行</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/usr/local/hadoop-2.7.3</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#更新设置</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
-<h3 id="配置启动hadoop">配置启动Hadoop</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="comment">#修改配置文件</span></span><br><span class="line">vim hadoop-env.sh</span><br><span class="line"><span class="comment">#第二十五行</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim core-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 指定HDFS老大（namenode）的通信地址 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">	    &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">	    &lt;value&gt;hdfs://hadoopbonc1:9000&lt;/value&gt;</span><br><span class="line">    &lt;!-- 指定hadoop运行时产生文件的存储路径 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#此处fs.defaultFS的vaule写本机主机名，即第一个value的hadoopbonc1换成主机名</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim hdfs-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 设置hdfs副本数量 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;1&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br></pre></td></tr></table></figure>
-<h3 id="配置ssh免密码登陆密码互通">配置SSH免密码登陆（密码互通）</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#配置前</span></span><br><span class="line">ssh localhost</span><br><span class="line"><span class="comment">#回车后，要求输入本机登陆密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#配置主机名</span></span><br><span class="line">vim /etc/sysconfig/network</span><br><span class="line"><span class="built_in">source</span> /etc/sysconfig/network</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置hosts</span></span><br><span class="line">vim /etc/hosts</span><br><span class="line">ssh-keygen</span><br><span class="line"></span><br><span class="line"><span class="comment">#把公钥和私钥复制到相应的节点（因为本机是单节点，所以复制到本机）</span></span><br><span class="line"><span class="comment">#此处的hadoopbonc1换成主机名</span></span><br><span class="line">ssh-copy-id hadoopbonc1</span><br></pre></td></tr></table></figure>
-<h3 id="hdfs启动与停止">hdfs启动与停止</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#第一次启动需要格式化，理由跟磁盘第一次使用需要格式化一样</span></span><br><span class="line"><span class="comment">#以后启动就不需要格式化</span></span><br><span class="line">hdfs namenode -format</span><br><span class="line"><span class="comment">#启动hdfs</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#在hadoop目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/locate/hadoop-2.7.3/</span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">./sbin/start-dfs.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现四行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:50070</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#若无响应则需要开放50070端口，则需开放50070端口</span></span><br><span class="line">firewall-cmd --zone=public --add-port=50070/tcp --permanent</span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment">#hdfs的停止</span></span><br><span class="line"><span class="comment">#同样在hadoop目录下</span></span><br><span class="line">./sbin/stop-dfs.sh</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="配置和启动yarn">配置和启动YARN</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#切换配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="built_in">mv</span> mapred-site.xml.template mapred-site.xml</span><br><span class="line">vim mapred-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 通知框架MR使用YARN --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;yarn&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim yarn-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- reducer取数据的方式是mapreduce_shuffle --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;mapreduce_shuffle&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#启动YARN</span></span><br><span class="line">start-yarn.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现六行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps、ResourceManager、NodeManager，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:8088</span></span><br><span class="line"><span class="comment">#若无响应，可参照上文中的开放50070端口，开放8088端口，只需将50070换为8088即可</span></span><br></pre></td></tr></table></figure>
-<h2 id="测试">测试</h2>
-<p>在本地新建一个文件，如在<code>/home/user/</code>下新建<code>words.txt</code>，内容如下
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hello world</span><br><span class="line">hello hadoop</span><br><span class="line">hello csdn</span><br><span class="line">hello</span><br></pre></td></tr></table></figure></p>
-<p>正式进行测试。命令如下： <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hdfs根目录下新建test目录</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">mkdir</span> /test</span><br><span class="line"></span><br><span class="line"><span class="comment">#查看hdfs根目录下的目录结构</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /</span><br><span class="line"></span><br><span class="line"><span class="comment">#将本地文件上传至/test/目录下</span></span><br><span class="line">bin/hdfs dfs -put /home/user/words.txt /test/</span><br><span class="line"></span><br><span class="line"><span class="comment">#运行wordcount</span></span><br><span class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /test/words.txt /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#在/test/目录下生成了一个名为out的文件目录，查看一下/out/目录下的文件</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#结果保存在part-r-00000，查看一下运行结果</span></span><br><span class="line">bin/hdfs fs -<span class="built_in">cat</span> /test/out/part-r-00000</span><br></pre></td></tr></table></figure></p>
-<h2 id="hdfs的常用操作命令">HDFS的常用操作命令</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#常用操作：</span></span><br><span class="line"><span class="comment">#HDFS shell</span></span><br><span class="line"><span class="comment">#查看帮助</span></span><br><span class="line">hadoop fs -<span class="built_in">help</span> &lt;cmd&gt;</span><br><span class="line"><span class="comment">#上传</span></span><br><span class="line">hadoop fs -<span class="built_in">cat</span> &lt;hdfs上的路径&gt;</span><br><span class="line"><span class="comment">#查看文件列表</span></span><br><span class="line">hadoop fs -<span class="built_in">ls</span> /</span><br><span class="line"><span class="comment">#下载文件</span></span><br><span class="line">hadoop fs -get &lt;hdfs上的路径&gt; &lt;linux上文件&gt;</span><br></pre></td></tr></table></figure>
-<p>下一篇：<a href="/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/" title="在Hadoop基础上Hive的安装">在Hadoop基础上Hive的安装</a></p>
-]]></content>
-      <categories>
-        <category>Environment</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>Hadoop</tag>
-        <tag>大数据</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Ubuntu20.04安装深度学习环境及配置个人环境小记</title>
-    <url>/hexo_blog/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</url>
-    <content><![CDATA[<p>Ubuntu20.04安装深度学习环境及配置个人环境小记</p>
-<span id="more"></span>
-<h3 id="目标配置环境">目标配置环境</h3>
-<p>系统为Ubuntu 20.04,
-显卡为3090，结合其他服务器的环境，准备配置如下环境</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nvidia-driver-460 <span class="comment"># 英伟达驱动，最低推荐版本为这个</span></span><br><span class="line">CUDA 11.2 (gcc 7) <span class="comment"># 30系显卡应该换为cuda11版本以上，20系显卡可以用cuda10.2版本</span></span><br><span class="line">cudnn 8.6.1</span><br><span class="line"></span><br><span class="line">pytorch 1.10</span><br><span class="line">torchvision 0.11.1</span><br><span class="line">tensorflow 2.5.0</span><br></pre></td></tr></table></figure>
-<h3 id="准备工作">准备工作</h3>
-<p>刚装的系统，啥也没有，所以按照个人习惯先安装一些软件。为方便起见，后文非特殊说明，均用root用户完成安装。可以直接复制下文至sh文件中，更改权限以直接运行。（请保证软件源可正常连接）</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一步: 安装一些必备的软件</span></span><br><span class="line">add-apt-repository ppa:graphics-drivers/ppa</span><br><span class="line">apt update -y &amp;&amp; apt upgrade -y</span><br><span class="line">apt install -y git autossh python3-pip vim tmux gpustat tree ranger build-essential gcc-7 g++-7 make curl htop ipython3 zhs</span><br><span class="line"><span class="comment"># gcc与g++降级</span></span><br><span class="line">update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100</span><br><span class="line">update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步：安装nvidia驱动</span></span><br><span class="line"><span class="comment"># 禁用nouveau</span></span><br><span class="line"><span class="built_in">chmod</span> 666 /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 增加相应配置, 若已追加则记得注释下面这一行命令</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;</span></span><br><span class="line"><span class="string">blacklist vga16fb</span></span><br><span class="line"><span class="string">blacklist nouveau</span></span><br><span class="line"><span class="string">blacklist rivafb</span></span><br><span class="line"><span class="string">blacklist rivatv</span></span><br><span class="line"><span class="string">blacklist nvidiafb</span></span><br><span class="line"><span class="string">&quot;</span></span><br><span class="line">&gt;&gt; /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 更新配置</span></span><br><span class="line">update-initramfs -u</span><br><span class="line">apt install nvidia-driver-460 -y <span class="comment"># 英伟达460驱动，按需更改</span></span><br></pre></td></tr></table></figure>
-<p>执行完成之后，重启服务器。使用<code>nvidia-smi</code>查看gpu是否正常，或使用安装好的<code>gpustat</code></p>
-<h3 id="cuda及cudnn安装">cuda及cudnn安装</h3>
-<h4 id="下载">下载</h4>
-<p>这里需要从官网下载安装包，cuda安装包可以直接通过命令下载。cudnn需要注册英伟达开发者账号，下载四个文件</p>
-<ul>
-<li><p><code>cudnn-11.2-linux-x64-v8.1.0.77</code></p></li>
-<li><p><code>libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
-<li><p><code>libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
-<li><p><code>libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda_11.2.2_460.32.03_linux.run &amp;&amp; <span class="built_in">chmod</span> 755 cuda_11.2.2_460.32.03_linux.run</span><br><span class="line"></span><br><span class="line">./cuda_11.2.2_460.32.03_linux.run</span><br></pre></td></tr></table></figure>
-<p>下载完直接运行，安装过程中需要把Drivers这项取消，因为自带了显卡驱动。</p>
-<h4 id="环境变量">环境变量</h4>
-<p>在/etc/bash.bashrc或/etc/profile前加入如下两行。如果profile不行就换bashrc。一般是编辑<code>vim ~/.bashrc</code>下，但为保证所有用户都生效，所以需要在<code>/etc</code>目录下编辑</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">export</span> PATH=/usr/local/cuda-11.2/bin<span class="variable">$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span></span><br><span class="line"><span class="built_in">export</span> LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64<span class="variable">$&#123;LD_LIBRARY_PATH:+:<span class="variable">$&#123;LD_LIBRARY_PATH&#125;</span>&#125;</span></span><br></pre></td></tr></table></figure>
-<p>至此，cuda安装完成</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 检测是否安装成功</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/cuda/samples/1_Utilities/deviceQuery</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./deviceQuery</span><br><span class="line"><span class="comment"># 最后一行输出，Result = PASS表示成功</span></span><br></pre></td></tr></table></figure>
-<h4 id="cudnn配置">cudnn配置</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -zxvf cudnn-11.2-linux-x64-v8.1.0.77.tgz </span><br><span class="line"><span class="built_in">cp</span> cuda/include/cudnn.h /usr/local/cuda/include </span><br><span class="line"><span class="built_in">cp</span> cuda/lib64/libcudnn* /usr/local/cuda/lib64  </span><br><span class="line"><span class="built_in">chmod</span> a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*  </span><br><span class="line"></span><br><span class="line">dpkg -i libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb      </span><br><span class="line">dpkg -i libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb  </span><br><span class="line">dpkg -i libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb </span><br><span class="line"></span><br><span class="line"><span class="comment"># cudnn检测，更高版本的cudnn需要把cudnn.h换成cudnn_version.h</span></span><br><span class="line"><span class="built_in">cat</span> /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2</span><br><span class="line"><span class="comment"># 利用程序检查，可能需要重启</span></span><br><span class="line"><span class="built_in">cd</span> /usr/src/cudnn_samples_v7/conv_sample</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./conv_sample</span><br><span class="line"><span class="comment"># 最后一行输出，Test passed！表示成功</span></span><br></pre></td></tr></table></figure>
-<p>通过<code>ldconfig</code>可检查动态链接库是否链接正确。若有输出，可以使用<code>ln -sf</code>重新链接。</p>
-<h3 id="python库安装">Python库安装</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pip3 install tensorflow-gpu==2.5.0 -i https://pypi.tuna.tsinghua.edu.cn/simple</span><br><span class="line">pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html</span><br><span class="line">pip3 install black tqdm flask sklearn</span><br></pre></td></tr></table></figure>
-<h4 id="检查是否安装成功">检查是否安装成功</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.cuda.is_available()) </span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.is_available()) </span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.__version__)</span><br><span class="line"><span class="built_in">print</span>(torch.version.cuda)</span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.version())</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br><span class="line">gpu_device_name = tf.test.gpu_device_name()</span><br><span class="line"><span class="built_in">print</span>(gpu_device_name)   </span><br><span class="line">tf.test.is_gpu_available() <span class="comment"># 输出True</span></span><br></pre></td></tr></table></figure>
-<p>若tf未检测到GPU，可以看输出过程中的哪些动态库文件加载失败，手动配置一下。</p>
-<p>至此，深度学习安装环境配置完成。</p>
-<h3 id="个人环境">个人环境</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建用户</span></span><br><span class="line">adduser xxx</span><br><span class="line"><span class="comment"># 输入配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 增加root权限</span></span><br><span class="line"><span class="built_in">chmod</span> u+w /etc/sudoers</span><br><span class="line">vim /etc/sudoers</span><br><span class="line">​```</span><br><span class="line">root    ALL=(ALL:ALL) ALL</span><br><span class="line"><span class="comment"># 增加一行</span></span><br><span class="line">xxx    ALL=(ALL:ALL) ALL</span><br><span class="line">​```</span><br><span class="line"><span class="built_in">chmod</span> u-w /etc/sudoers</span><br></pre></td></tr></table></figure>
-<h4 id="ssh文件配置">ssh文件配置</h4>
-<p>方便起见直接把其他服务器的<code>.ssh</code>文件夹下的<code>id_rsa</code>，<code>id_rsa.pub</code>和<code>authorized_keys</code>复制过来。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 复制完成之后，修改对应权限才能使用</span></span><br><span class="line"><span class="built_in">chmod</span> 755 ~/.ssh/  </span><br><span class="line"><span class="built_in">chmod</span> 600 ~/.ssh/id_rsa ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys</span><br><span class="line"><span class="built_in">chmod</span> 644 ~/.ssh/known_hosts</span><br></pre></td></tr></table></figure>
-<h4 id="oh-my-zsh">oh-my-zsh</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载并安装</span></span><br><span class="line">wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装插件</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-autosuggestions <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-autosuggestions</span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-syntax-highlighting.git <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-syntax-highlighting</span><br><span class="line"></span><br><span class="line"><span class="comment"># vim ~/.zshrc, 最后一行加入如下配置</span></span><br><span class="line">plugins=(</span><br><span class="line">        git</span><br><span class="line">        zsh-autosuggestions</span><br><span class="line">        zsh-syntax-highlighting</span><br><span class="line">)</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/oh-my-zsh.sh</span><br></pre></td></tr></table></figure>
-<h4 id="配置vim">配置vim</h4>
-<p>编辑<code>.vimrc</code></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">set</span> number</span><br><span class="line"></span><br><span class="line">syntax on</span><br><span class="line">colorscheme darkblue</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">syntax <span class="built_in">enable</span></span><br><span class="line">filetype plugin on</span><br><span class="line"><span class="built_in">let</span> g:go_disable_autoinstall = 0</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_functions = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_methods = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_structs = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_operators = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_build_constraints = 1</span><br><span class="line"><span class="string">&quot; &quot;</span> syntax on</span><br><span class="line"><span class="string">&quot;&quot;</span> filetype plugin indent on</span><br><span class="line"><span class="built_in">let</span> g:pymode_python = <span class="string">&#x27;python3&#x27;</span></span><br><span class="line"><span class="built_in">set</span> number</span><br><span class="line"><span class="built_in">set</span> cin</span><br><span class="line"><span class="built_in">set</span> sw=8</span><br><span class="line"><span class="built_in">set</span> ts=8</span><br><span class="line"><span class="built_in">set</span> sm</span><br><span class="line">colorscheme darkblue</span><br><span class="line"><span class="built_in">set</span> viminfo=<span class="string">&#x27;10,\&quot;100,:20,%,n~/.viminfo </span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">au BufReadPost * if line(&quot;&#x27;</span>\&quot;<span class="string">&quot;) &gt; 0|if line(&quot;</span><span class="string">&#x27;\&quot;&quot;) &lt;= line(&quot;$&quot;)|exe(&quot;norm &#x27;</span>\&quot;<span class="string">&quot;)|else|exe &quot;</span>norm $<span class="string">&quot;|endif|endif</span></span><br></pre></td></tr></table></figure>
-<h4 id="远程连接">远程连接</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">autossh -gfCNR 远程端口:localhost:22（默认本地ssh端口） 公网用户名@公网IP</span><br></pre></td></tr></table></figure>
-<p>需要提前配好免密登录，远程端口在公网服务器上需要开放防火墙。</p>
-<p>远程连接登录<code>ssh 服务器用户@公网IP -p 远程端口</code></p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span> <span class="keyword">ALL</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br></pre></td></tr></table></figure>
+<ol start="28" type="1">
+<li><p>用WHERE替代ORDER BY</p>
+<p>ORDER BY 子句只在两种严格的条件下使用索引.</p>
+<p>ORDER BY中所有的列必须包含在相同的索引中并保持在索引中的排列顺序.</p>
+<p>ORDER BY中所有的列必须定义为非空.</p>
+<p>WHERE子句使用的索引和ORDER BY子句中所使用的索引不能并列.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引不被使用)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">ORDER</span> <span class="keyword">BY</span> DEPT_TYPE</span><br><span class="line">高效: (使用索引)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> DEPT_TYPE <span class="operator">&gt;</span> <span class="number">0</span></span><br></pre></td></tr></table></figure>
+<ol start="29" type="1">
+<li><p>避免改变索引列的类型</p>
+<p>当比较不同数据类型的数据时,
+ORACLE自动对列进行简单的类型转换.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 假设 EMPNO是一个数值类型的索引列.</span><br><span class="line"># 低效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="string">&#x27;123&#x27;</span></span><br><span class="line"># 高效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="number">123</span></span><br></pre></td></tr></table></figure>
+<ol start="30" type="1">
+<li><p>需要当心的WHERE子句</p>
+<p>某些SELECT 语句中的WHERE子句不使用索引. 这里有一些例子.</p>
+<ol type="1">
+<li><code>!=</code> 将不使用索引. 记住, 索引只能告诉你什么存在于表中,
+而不能告诉你什么不存在于表中.</li>
+<li><code>¦ ¦</code>是字符连接函数. 就象其他函数那样, 停用了索引.</li>
+<li><code>+</code>是数学函数. 就象其他数学函数那样, 停用了索引.</li>
+<li>相同的索引列不能互相比较,这将会启用全表扫描.</li>
+</ol></li>
+<li><ol type="a">
+<li><p>如果检索数据量超过30%的表中记录数.使用索引将没有显著的效率提高.</p></li>
+<li><p>在特定情况下, 使用索引也许会比全表扫描慢,
+但这是同一个数量级上的区别.
+而通常情况下,使用索引比全表扫描要块几倍乃至几千倍!</p></li>
+</ol></li>
+<li><p>避免使用耗费资源的操作</p>
+<p>带有DISTINCT,UNION,MINUS,INTERSECT,ORDER
+BY的SQL语句会启动SQL引擎执行耗费资源的排序(SORT)功能.
+DISTINCT需要一次排序操作, 而其他的至少需要执行两次排序. 通常, 带有UNION,
+MINUS , INTERSECT的SQL语句都可以用其他方式重写.
+如果你的数据库的SORT_AREA_SIZE调配得好, 使用UNION , MINUS,
+INTERSECT也是可以考虑的, 毕竟它们的可读性很强。</p></li>
+<li><p>优化GROUP BY</p>
+<p>提高GROUP BY 语句的效率, 可以通过将不需要的记录在GROUP BY
+之前过滤掉.下面两个查询返回相同结果但第二个明显就快了许多.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br><span class="line"><span class="keyword">HAVING</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">WHERE</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br></pre></td></tr></table></figure>
+<pre><code>​</code></pre>
 ]]></content>
       <categories>
-        <category>Linux</category>
+        <category>Database</category>
       </categories>
       <tags>
-        <tag>CUDA</tag>
-        <tag>PyTorch</tag>
-        <tag>Ubuntu</tag>
-        <tag>Tensorflow</tag>
-        <tag>CUDNN</tag>
+        <tag>数据库</tag>
+        <tag>SQL</tag>
       </tags>
   </entry>
   <entry>
-    <title>在Hadoop基础上Hive的安装</title>
-    <url>/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Hive在Hadoop上的安装及一些基本的Hive操作</p>
+    <title>NLTK阅读笔记Ⅰ</title>
+    <url>/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
 <span id="more"></span>
-<p>写在开头 环境 Hadoop单机 Centos7 Hadoop-2.7.3
-hadoop位置：/usr/loacl/hadoop</p>
-<p>参考文章 <a
-href="http://yanliu.org/2015/08/13/Hadoop%E9%9B%86%E7%BE%A4%E4%B9%8BHive%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">Hadoop集群之Hive安装配置</a></p>
-<h2 id="下载hive">下载Hive</h2>
-<p>下载源码包 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hadoop目录下操作</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop</span><br><span class="line"></span><br><span class="line"><span class="comment">#用wget下载</span></span><br><span class="line">wget http://mirrors.cnnic.cn/apache/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#也可以在图形界面下下载之后，上传压缩包</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#解压缩包</span></span><br><span class="line">tar -zxvf apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置环境变量</span></span><br><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#位置在之前配置的变量之后，大概12行左右，因为之前配置了jdk和hadoop变量</span></span><br><span class="line"><span class="built_in">export</span> HIVE_HOME=/usr/local/hadoop/apache-hive-1.2.2-bin</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HIVE_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#适当位置添加&quot;$HIVE_HOME/bin:&quot;</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#使文件生效</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure></p>
-<h2 id="安装mysql">安装MySQL</h2>
-<p>MariaDB数据库管理系统是MySQL的一个分支，主要由开源社区在维护，采用GPL授权许可。开发这个分支的原因之一是：甲骨文公司收购了MySQL后，有将MySQL闭源的潜在风险，因此社区采用分支的方式来避开这个风险。MariaDB的目的是完全兼容MySQL，包括API和命令行，使之能轻松成为MySQL的代替品。</p>
-<p>在Centos7下，使用命令安装mysql，会安装成mariadb。</p>
-<p>在这里先介绍安装MariaDB，原因如上。</p>
-<h3 id="安装mariadb">安装MariaDB</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#安装</span></span><br><span class="line">yum install mariadb-server mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">systemctl start mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入MySQL</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下，可能有点不习惯</span></span><br><span class="line">MariaDB [(none)]&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#P.S.</span></span><br><span class="line"><span class="comment">#相关命令</span></span><br><span class="line">systemctl start mariadb <span class="comment">#启动MariaDB</span></span><br><span class="line">systemctl stop mariadb <span class="comment">#停止MariaDB</span></span><br><span class="line">systemctl restart mariadb <span class="comment">#重启MariaDB</span></span><br><span class="line">systemctl <span class="built_in">enable</span> mariadb <span class="comment">#设置开机启动</span></span><br></pre></td></tr></table></figure>
-<h3 id="正式安装mysql">正式安装MySQL</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下载，在这里使用的是命令行下载，也建议在图形界面下载，然后上传至服务器</span></span><br><span class="line">wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用rpm安装</span></span><br><span class="line">rpm -ivh mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用yum安装mysql-community-server</span></span><br><span class="line">yum install mysql-community-server</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动服务</span></span><br><span class="line">service mysqld start</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入mysql，第一次进入无密码</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下</span></span><br><span class="line">mysql&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="配置mysql">配置MySQL</h3>
-<p>编码配置 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件，若未安装vim，建议先使用命令yum install vim安装vim</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"></span><br><span class="line"><span class="comment">#最后加上编码配置</span></span><br><span class="line">[mysql]</span><br><span class="line">default-character-set =utf8</span><br><span class="line"></span><br><span class="line"><span class="comment">#此处字符编码必须和/usr/share/mysql/charsets/Index.xml中一致。</span></span><br><span class="line"><span class="comment">#不过一般情况下使用的都是utf8</span></span><br></pre></td></tr></table></figure></p>
-<p>设置密码 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下面三种方法需要进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#方法一</span></span><br><span class="line">mysql&gt;insert into user(host,user,password) values(<span class="string">&#x27;%&#x27;</span>,<span class="string">&#x27;user_name&#x27;</span>,password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法二</span></span><br><span class="line">mysql&gt;<span class="built_in">set</span> password <span class="keyword">for</span> user_name = password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法三</span></span><br><span class="line">mysql&gt;grant all on *.* to user_name@% identified by <span class="string">&quot;password&quot;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">#下面这一种方法可直接在shell下设置密码</span></span><br><span class="line">mysqladmin -u root password <span class="string">&quot;password&quot;</span></span><br></pre></td></tr></table></figure> 远程连接</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#把在所有数据库的所有表的所有权限赋值给位于所有IP地址的root用户。</span></span><br><span class="line">mysql&gt; grant all privileges on *.* to root@<span class="string">&#x27;%&#x27;</span>identified by <span class="string">&quot;password&quot;</span>;</span><br></pre></td></tr></table></figure>
-<p>P.S.上文引号中的user_name表示数据库的用户名，password表示对应用户的密码。即这两项是由读者自行定义的。</p>
-<h3 id="常见问题及解决方案">常见问题及解决方案</h3>
-<h4 id="mysql创建用户失败">Mysql创建用户失败</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># mysql在my.ini的配置文件中设置了严格模式，所以我们需要进行修改</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第一步，寻找配置文件</span></span><br><span class="line">whereis my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步，根据上一步结果进行vim编辑</span></span><br><span class="line">vim /../my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第三步，在vim里面搜索sql-mode,删除STRICT_TRANS_TABLES，保存退出即可</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第四步，保险起见，使刚刚的配置文件立即生效</span></span><br><span class="line"><span class="built_in">source</span> /../my.ini</span><br></pre></td></tr></table></figure>
-<h4 id="支持中文">支持中文</h4>
+<h2 id="写在开头">写在开头</h2>
+<p>在这里主要是对NLP的相关知识做一个整理和对NLTK模块的介绍，书中Python基础内容在这里不做介绍，如书中有我认为值得介绍的Python写法，我会进行说明。</p>
+<p>由于书中nltk是老版本的问题，语法上存在一些变动，在这里也会进行修正。可能由于书是译本的原因（当然也可能原作者自己的失误），部分代码有些错误，在这里我也进行了校正。如果您发现了我的文章中有需要改正或改进的地方，欢迎在评论区提出。</p>
+<p>最后十分感谢原作者的贡献和译者的翻译，强烈推荐读者亲自阅读此书。</p>
+<p>nltk版本是3.2.3，<a href="http://www.nltk.org/">官网</a></p>
+<p>python版本3.6.1</p>
+<p>数据、PDF和一些重要的代码已经放在了Github上了。<a
+href="https://github.com/wnma3mz/Nltk_Study">github地址</a></p>
+<h2 id="第一章语言处理与python">第一章：语言处理与Python</h2>
+<p>NLTK库的安装，在这里不做介绍，强烈建议直接使用Anaconda环境。<a
+href="https://www.anaconda.com/download/">官网下载链接</a>。本文是直接使用了Anaconda环境。</p>
+<p>数据的获取，这里nltk有官方提供的文本数据，可以直接使用<code>nltk.download()</code>打开图形界面，下载语料集<code>book</code>。由于使用这个方法下载速度比较慢的原因，我在这里Github上提供了数据集<code>nltk_data</code>，下载之后，移动到<code>nltk.download()</code>原本的下载目录下，之后再运行<code>nltk.download()</code>就不需要下载数据集而是直接解压数据集，速度会快上很多。<a
+href="https://github.com/wnma3mz/Nltk_Study">下载链接</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk模块</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 导入基本语料集(不需要额外下载)，包含text1到text9变量，可以直接输出这些变量</span></span><br><span class="line"><span class="keyword">from</span> nltk.book <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本。这里表示找到&quot;monstrous&quot;所包含的句子，并且输出上下文</span></span><br><span class="line">text1.concordance(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本出现在相似的上下文中</span></span><br><span class="line">text1.similar(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索两个及两个以上共同词的上下文</span></span><br><span class="line">text2.common_contexts([<span class="string">&quot;monstrous&quot;</span>, <span class="string">&quot;very&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画一张离散图表示这些词出现在文本中的位置，输出见下图</span></span><br><span class="line">text4.dispersion_plot([<span class="string">&quot;citizens&quot;</span>, <span class="string">&quot;democracy&quot;</span>, <span class="string">&quot;freedom&quot;</span>, <span class="string">&quot;duties&quot;</span>, <span class="string">&quot;America&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于文本，随机生成一些文本</span></span><br><span class="line">text3.generate()</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-1.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 有序字典，按词频从高到低排序</span></span><br><span class="line">fdist1 = FreqDist(text1)</span><br><span class="line"><span class="comment"># 选出词频最高的50个词</span></span><br><span class="line">fdist1.keys()[:<span class="number">50</span>]</span><br><span class="line"><span class="comment"># 某个词出现的频数</span></span><br><span class="line">fdist1[<span class="string">&#x27;whale&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># text1中词频最高的50个单词，进行绘图，输出见下图</span></span><br><span class="line">fdist1.plot(<span class="number">50</span>, cumulative=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># text1中只出现过一次的单词</span></span><br><span class="line">fdist1.hapaxes()</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-3.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 词语搭配，双连词(bigrams)</span></span><br><span class="line">nltk.bigrams([<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>, <span class="string">&#x27;done&#x27;</span>])</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[(<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>),(<span class="string">&#x27;is&#x27;</span>,<span class="string">&#x27;said&#x27;</span>),(<span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>), (<span class="string">&#x27;than&#x27;</span>,<span class="string">&#x27;done&#x27;</span>)]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文本中单个词的频率预期得到的更频繁出现的双连词</span></span><br><span class="line">text4.collocations()</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 与nltk自带的聊天机器人系统对话</span></span><br><span class="line"><span class="comment"># 导入模块</span></span><br><span class="line"><span class="keyword">from</span> nltk.chat <span class="keyword">import</span> chatbots</span><br><span class="line"><span class="comment"># 选择一个聊天机器人并开始对话</span></span><br><span class="line">chatbots()</span><br></pre></td></tr></table></figure>
+<p>补充部分：</p>
+<table>
+<colgroup>
+<col style="width: 7%" />
+<col style="width: 92%" />
+</colgroup>
+<thead>
+<tr>
+<th style="text-align: center;">text</th>
+<th style="text-align: center;">文本名</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style="text-align: center;">text1</td>
+<td style="text-align: center;">《白鲸记》（Moby Dick by Herman Melville
+1851）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text2</td>
+<td style="text-align: center;">《理智与情感》（Sense and Sensibility by
+Jane Austen 1811）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text3</td>
+<td style="text-align: center;">《创世纪》（The Book of Genesis）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text4</td>
+<td style="text-align: center;">《就职演说语料库》（Inaugural Address
+Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text5</td>
+<td style="text-align: center;">《NPS聊天语料库》（Chat Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text6</td>
+<td style="text-align: center;">《巨蟒与圣杯》（Monty Python and the
+Holy Grail）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text7</td>
+<td style="text-align: center;">《华尔街日报》（Wall Street
+Journal）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text8</td>
+<td style="text-align: center;">《个人文集》（Personals Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text9</td>
+<td style="text-align: center;">《周四的男人》（The Man Who Was Thursday
+by G . K . Chestert）</td>
+</tr>
+</tbody>
+</table>
+<p>概念补充：</p>
 <ul>
-<li><p>Centos7</p>
-<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 编辑文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加修改如下代码</span></span><br><span class="line">[client]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-character-set=utf8</span><br><span class="line"></span><br><span class="line">[mysqld]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-storage-engine=INNODB</span><br><span class="line">character-set-server=utf8</span><br><span class="line">collation-server=utf8_general_ci</span><br><span class="line"></span><br><span class="line">[mysql]</span><br><span class="line">no-auto-rehash</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 保存退出，重启服务</span></span><br><span class="line"><span class="comment"># 重新登陆mysql检查是否成功，方法见下</span></span><br></pre></td></tr></table></figure></p></li>
-<li><p>Ubuntu16.04</p>
-<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> vim /etc/mysql/mysql.conf.d/mysqld.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加以下内容，如果不存在[xx]自行增加</span></span><br><span class="line">[mysqld]</span><br><span class="line">character_set_server=utf8</span><br><span class="line">[mysql]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysql.server]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysqld_safe]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[client]</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 重启mysql</span></span><br><span class="line">service mysql restart</span><br><span class="line"><span class="comment"># 进入mysql查看配置参数</span></span><br><span class="line">mysql -uroot -p</span><br><span class="line"><span class="comment"># 查看database 的value变为utf8即可</span></span><br><span class="line">&gt;show variables like <span class="string">&#x27;%character%&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment"># p.s.如果之前创建了表，表编码不会改变</span></span><br></pre></td></tr></table></figure></p></li>
+<li>词类型：一个词在一个文本中独一无二的出现形式或拼写。也就是说，这个词在词汇表中是唯一的。</li>
+<li>频率分布：</li>
 </ul>
-<h4
-id="mysql报错1366---incorrect-integer-value-xxxxxxx-at-row-1">MySQL报错“1366
-- Incorrect integer value: '' XXXXXXX' at row 1 ”</h4>
-<p>修改方法:（两种,建议第二种）</p>
-<ol type="1">
-<li><p>命令行。set names gbk；(此为设置通信编码)</p></li>
-<li><p><code>my.cnf</code> 中查找sql-mode</p></li>
-</ol>
-<p>将 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 修改为 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 重启mysql后即可</p>
-<h4 id="关于my.cnf和my.ini的说明">关于my.cnf和my.ini的说明</h4>
-<p>my.cnf常见于Linux系统，my.ini常见与windows系统，二者都是属于mysql的配置文件。一般好像在一个系统下就是只出现一种配置文件，具体区别没有深入了解，修改配置文件，根据自己的系统进行查找修改配置文件即可</p>
-<h4 id="mysql交互环境自动补全">Mysql交互环境自动补全</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 修改配置文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在[mysql]部分添加auto-rehash</span></span><br><span class="line"><span class="comment"># 保存退出，重启</span></span><br><span class="line">service mysqld restart</span><br></pre></td></tr></table></figure>
-<h2 id="配置hive">配置Hive</h2>
-<p>我们之前在hadoop目录下安装了Hive，位置为<code>/usr/local/hadoop/apache-hive-1.2.1-bin</code></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入hive配置目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/conf</span><br><span class="line"></span><br><span class="line"><span class="comment">#修改hive-default.xml.template</span></span><br><span class="line"><span class="comment">#首先复制</span></span><br><span class="line"><span class="built_in">cp</span> hive-default.xml.template hive-default.xml</span><br><span class="line"><span class="comment">#修改文件</span></span><br><span class="line">vim hive-default.xml</span><br><span class="line"></span><br><span class="line"><span class="comment"># 1. 第一步将&lt;configuration&gt;&lt;/configuration&gt;中内容删除大概是18-3908行</span></span><br><span class="line"><span class="comment"># vim删除命令--&gt;:18,3908d</span></span><br><span class="line"><span class="comment"># 2.将下面&lt;configuration&gt;&lt;/configuration&gt;中内容复制进去,分别将3306喝9083前的user_name改为当前的用户名</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionURL&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;jdbc:mysql://user_name:3306/hive?createDatabaseIfNotExist=<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;JDBC connect string <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionDriverName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;com.mysql.jdbc.Driver&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;Driver class name <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionUserName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;value&gt;</span><br><span class="line">        &lt;description&gt;username to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionPassword&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;password to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">    &lt;name&gt;hive.metastore.uris&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;thrift://user_name:9083&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
-<h2 id="下载jdbc">下载JDBC</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#命令行下载,也可以使用图形界面上传文件</span></span><br><span class="line"> wget http://cdn.mysql.com/Downloads/Connector-J/mysql-connector-java-5.1.36.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#将文件复制进Hive的lib目录下，原因，Hive的自带的那个版本低，可能失效</span></span><br><span class="line"> <span class="built_in">cp</span> mysql-connector-java-5.1.33-bin.jar /usr/local/hadoop/apache-hive-1.2.1-bin/lib/</span><br></pre></td></tr></table></figure>
-<p>启动Hive <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hive --service metastore &amp;</span><br><span class="line">jps</span><br><span class="line"><span class="comment">#结果会多出一个进程</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#进入hive目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/bin</span><br><span class="line"><span class="comment">#启动hive，可能有点慢</span></span><br><span class="line">hive</span><br><span class="line"></span><br><span class="line"><span class="comment">#若出现hive的命令行即代表成功，如下</span></span><br><span class="line">hive&gt;</span><br></pre></td></tr></table></figure></p>
-<h2 id="常见问题及解决方案-1">常见问题及解决方案</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">Logging initialized using configuration <span class="keyword">in</span> jar:file:/home/hadoop/apache-hive-1.2.1-bin/lib/hive-common-1.2.1.jar!/hive-log4j.properties</span><br><span class="line">Exception <span class="keyword">in</span> thread <span class="string">&quot;main&quot;</span> java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient</span><br><span class="line">        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)</span><br></pre></td></tr></table></figure>
-<p>解决方法，直接关闭防火墙</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#这里的系统为Centos7，所以使用此命令</span></span><br><span class="line">systemctl stop firewalld</span><br></pre></td></tr></table></figure>
-<p>hive metastore 启动出错解决 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看与hive相关进程是否启动</span></span><br><span class="line">ps -ef | grep hive</span><br><span class="line"><span class="comment"># kill相关进程，为进程号</span></span><br><span class="line"><span class="built_in">kill</span> num</span><br><span class="line"><span class="comment"># 重新启动</span></span><br><span class="line">./hive</span><br></pre></td></tr></table></figure></p>
-<p>更多问题见此文章 <a
-href="http://blog.csdn.net/freedomboy319/article/details/44828337">Hive常见问题汇总</a></p>
-<h2 id="hive的学习笔记">Hive的学习笔记</h2>
-<h3 id="新建表">新建表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 新建一张表，名为“test”，里面有“name”、“id”两类，分别是“string”、“int”的数据类型，以“|”隔开一列，表是作为textfile的。</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">CREATE</span> <span class="keyword">TABLE</span> test (name String, id <span class="type">int</span>) <span class="type">ROW</span> FORMAT DELIMITED FIELDS TERMINATED <span class="keyword">BY</span> <span class="string">&#x27;|&#x27;</span> STORED <span class="keyword">AS</span> TEXTFILE;</span><br></pre></td></tr></table></figure>
-<h3 id="加载表">加载表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 从本地的/home/user/test.txt文件，将数据加载进test这个表</span></span><br><span class="line">hive<span class="operator">&gt;</span> LOAD DATA <span class="keyword">LOCAL</span> INPATH <span class="string">&#x27;/home/user/test.txt&#x27;</span> OVERWRITE <span class="keyword">INTO</span> <span class="keyword">TABLE</span> test;</span><br></pre></td></tr></table></figure>
-<h3 id="关联表">关联表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 将两张表通过一个或多个字段关联在一起</span></span><br><span class="line"><span class="comment">-- 将test_a和test_b表的id字段关联在一起，再加上“name”字段，组成一个新表</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> test_a.id, test_b.name <span class="keyword">FROM</span> test_a, test_b <span class="keyword">JOIN</span> test_b <span class="keyword">ON</span> (test_a.id <span class="operator">=</span> test_b.id);</span><br></pre></td></tr></table></figure>
-<h3 id="保存表">保存表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 由于hive下执行任务之后，并不会保存数据，所以我们使用INSERT命令来保存命令</span></span><br><span class="line"><span class="comment">-- 插入数据到test表中</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">TABLE</span> test</span><br><span class="line">     <span class="operator">&gt;</span> ...</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 保存表到hdfs上,保存输出结果到hdfs下的/out/目录下</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">LOCAL</span> DIRECTORY &quot;/out/&quot;</span><br><span class="line">    <span class="operator">&gt;</span> ...</span><br></pre></td></tr></table></figure>
-<h3 id="排序问题">排序问题</h3>
-<ol type="1">
-<li>order by</li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li><p>sort by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test SORT <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
-<li><p>distribute by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> name DISTRIBUTE <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
-<li><p>DISTRIBUTE BY with SORT BY DISTRIBUTE BY 和 GROUP BY
-有点类似，DISTRIBUTE BY控制reduce如何处理数据，而SORT
-BY控制reduce中的数据如何排序。 注意：hive要求DISTRIBUTE BY语句出现在SORT
-BY语句之前。</p></li>
-<li><p>Cluster By</p></li>
-</ol>
-<p>cluster by 除了具有 distribute by 的功能外还兼具 sort by 的功能。
-默认升序排序，但DISTRIBUTE BY的字段和SORT
-BY的字段必须相同，且不能指定排序规则。</p>
-<p>总结：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-2.png" /></p>
 <ul>
-<li><code>ORDER BY</code>是全局排序，但在数据量大的情况下，花费时间会很长</li>
-<li><code>SORT BY</code>是将<code>reduce</code>的单个输出进行排序，不能保证全局有序</li>
-<li><code>DISTRIBUTE BY</code>
-可以按指定字段将数据划分到不同的<code>reduce</code>中</li>
-<li>当<code>DISTRIBUTE BY</code> 的字段和
-<code>SORT BY</code>的字段相同时，可以用<code>CLUSTER BY</code>来代替<code>DISTRIBUTE BY with SORT BY</code>。</li>
+<li>FreqDist的API</li>
+</ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-4.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-5.png" /></p>
+<ul>
+<li>词意消歧：需要算出特定上下文中的词被赋予的是哪个意思。单词可能存在相同/相近的含义，此时需要根据上下文来推断单词在此情景下的含义。</li>
+<li>指代消解（anaphora resolution）：确定代词或名词短语指的是什么</li>
+<li>先行词：代词可能代表的对象</li>
+<li>语义角色标注（semantic role
+labeling）：确定名词短语如何与动词相关联（如施事，受事，工具等）</li>
+<li>自动生成语言：需要解决自动语言理解。弄清楚词的含义、动作的主语以及代词的先行词是理解句子含义的步骤</li>
+<li>机器翻译（MT）：难点，一个给定的词可能存在几种不同的解释。</li>
+<li>文本对齐：根据一个网站发布的多种语言版本，来自动配对组成句子</li>
+<li>人机对话系统：图灵测试。</li>
+<li>文本含义识别（Recognizing Textual Entailment 简称
+RTE）：根据假定的一些条件，来推断给出的一句话是否正确。</li>
+</ul>
+<h2
+id="第二章获得文本语料和词汇资源">第二章：获得文本语料和词汇资源</h2>
+<p><a href="http://www.gutenberg.org/">古腾堡语料库</a></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-1.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk的古腾堡语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> gutenberg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看这个语料库中的所有txt文件名</span></span><br><span class="line">gutenberg.fileids()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中其中的一个</span></span><br><span class="line">emma = gutenberg.words(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 查看它包含的词汇个数</span></span><br><span class="line"><span class="built_in">len</span>(emma)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果要使用第一章中concordance()这样的命令，就必须要将数据放到nltk.Text对象中</span></span><br><span class="line">emma = nltk.Text(emma)</span><br><span class="line">emma.concordance(<span class="string">&quot;suprprize&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用words获取的是经过分隔成标识符的文本。使用sents函数则可以获取以句子进行划分后的数据。具体区别可以运行看看，请读者自行对比。</span></span><br><span class="line">emma_sents = gutenberg.sents(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取原始文本。一个字符串</span></span><br><span class="line">emma_raw = gutenberg.raw(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网络和聊天文本</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> webtext</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> nps_chat</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库中所有的类别</span></span><br><span class="line">brown.categories()</span><br><span class="line"><span class="comment"># words函数根据类别选择数据</span></span><br><span class="line">brown.words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">[<span class="string">&#x27;The&#x27;</span>, <span class="string">&#x27;Fulton&#x27;</span>, <span class="string">&#x27;County&#x27;</span>, <span class="string">&#x27;Grand&#x27;</span>, <span class="string">&#x27;Jury&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># words函数根据文件名选择数据</span></span><br><span class="line">brown.words(fileids=[<span class="string">&#x27;cg22&#x27;</span>])</span><br><span class="line">[<span class="string">&#x27;Does&#x27;</span>, <span class="string">&#x27;our&#x27;</span>, <span class="string">&#x27;society&#x27;</span>, <span class="string">&#x27;have&#x27;</span>, <span class="string">&#x27;a&#x27;</span>, <span class="string">&#x27;runaway&#x27;</span>, <span class="string">&#x27;,&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># sents函数根据类别选择数据</span></span><br><span class="line">brown.sents(categories=[<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;editorial&#x27;</span>, <span class="string">&#x27;reviews&#x27;</span>])</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="comment"># 带条件的频率分布函数</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 取出brown语料库中所有类别中的所有词。每个类别对应该类别下的所有词。(类别，所有词)</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (genre, word)</span><br><span class="line">        <span class="keyword">for</span> genre <span class="keyword">in</span> brown.categories()</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> brown.words(categories=genre))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定一部分文本类别(genres)</span></span><br><span class="line">genres = [<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;religion&#x27;</span>, <span class="string">&#x27;hobbies&#x27;</span>, <span class="string">&#x27;science_fiction&#x27;</span>, <span class="string">&#x27;romance&#x27;</span>, <span class="string">&#x27;humor&#x27;</span>]</span><br><span class="line"><span class="comment"># 指定一些情态动词(modals)</span></span><br><span class="line">modals = [<span class="string">&#x27;can&#x27;</span>, <span class="string">&#x27;could&#x27;</span>, <span class="string">&#x27;may&#x27;</span>, <span class="string">&#x27;might&#x27;</span>, <span class="string">&#x27;must&#x27;</span>, <span class="string">&#x27;will&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出一张表格。genres与modals的对应数量关系</span></span><br><span class="line">cfd.tabulate(conditions=genres, samples=modals)</span><br><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">                  can could   may might  must  will</span></span><br><span class="line"><span class="string">           news    93    86    66    38    50   389</span></span><br><span class="line"><span class="string">       religion    82    59    78    12    54    71</span></span><br><span class="line"><span class="string">        hobbies   268    58   131    22    83   264</span></span><br><span class="line"><span class="string">science_fiction    16    49     4    12     8    16</span></span><br><span class="line"><span class="string">        romance    74   193    11    51    45    43</span></span><br><span class="line"><span class="string">          humor    16    30     8     8     9    13</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br></pre></td></tr></table></figure>
+<p>通过输出，我们很容易发现，在<code>news</code>类别里面用的最多的情态动词是<code>will</code>。当然，还有一些结论就由读者自行发现了。</p>
+<p>关于其他语料库的信息挖掘，在这里就不赘述了。书中也只是浅显的用了之前用过的一些方法。下面展示一些我认为比较有意思的代码及输出。</p>
+<p>官网提供了如何访问NLTK语料库的其他例子，<a
+href="http://www.nltk.org/howto/corpus.html">链接</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> inaugural</span><br><span class="line"></span><br><span class="line"><span class="comment"># 先选出inaugural语料库中的所有字段的对应所有词汇，再统计出文本中以`america`和`citizen`开头的词在每个年份（filed）出现次数。（`america`或`citizen`, 年份）</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (target, fileid[:<span class="number">4</span>])</span><br><span class="line">        <span class="keyword">for</span> fileid <span class="keyword">in</span> inaugural.fileids()</span><br><span class="line">        <span class="keyword">for</span> w <span class="keyword">in</span> inaugural.words(fileid)</span><br><span class="line">        <span class="keyword">for</span> target <span class="keyword">in</span> [<span class="string">&#x27;america&#x27;</span>, <span class="string">&#x27;citizen&#x27;</span>]</span><br><span class="line">        <span class="keyword">if</span> w.lower().startswith(target))</span><br><span class="line"><span class="comment"># 绘图</span></span><br><span class="line">cfd.plot()</span><br></pre></td></tr></table></figure>
+<p>以<code>america</code>或<code>citizen</code>开始的词随时间（年份）的演变趋势</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-2.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># udhr语料库中不同语言版本的字长差异（彩色图）</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> udhr</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选取比较的语言</span></span><br><span class="line">languages = [<span class="string">&#x27;Chickasaw&#x27;</span>, <span class="string">&#x27;English&#x27;</span>, <span class="string">&#x27;German_Deutsch&#x27;</span>, <span class="string">&#x27;Greenlandic_Inuktikut&#x27;</span>, <span class="string">&#x27;Hungarian_Magyar&#x27;</span>, <span class="string">&#x27;Ibibio_Efik&#x27;</span>]</span><br><span class="line"><span class="comment"># 提取不同语言的数据</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (lang, <span class="built_in">len</span>(word))</span><br><span class="line">        <span class="keyword">for</span> lang <span class="keyword">in</span> languages</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> udhr.words(lang + <span class="string">&#x27;-Latin1&#x27;</span>))</span><br><span class="line"></span><br><span class="line">cfd.plot(cumulative=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
+<p>累积字长分布： “世界人权宣言”的6个翻译版本 <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-3.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 载入自己的语料库，文本文件</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> PlaintextCorpusReader</span><br><span class="line"><span class="comment"># 设置语料库的路径</span></span><br><span class="line">corpus_root = <span class="string">&#x27;/usr/share/dict&#x27;</span></span><br><span class="line"><span class="comment"># 解析路径下的所有文件</span></span><br><span class="line">wordlists = PlaintextCorpusReader(corpus_root, <span class="string">&#x27;.*&#x27;</span>)</span><br><span class="line"><span class="comment"># 语料库中所有的文件名</span></span><br><span class="line">wordlists.fileids()</span><br><span class="line"><span class="comment"># 语料库中`connectives`中的单词</span></span><br><span class="line">wordlists.words(<span class="string">&#x27;connectives&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 本地个人的语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> BracketParseCorpusReader</span><br><span class="line">corpus_root = <span class="string">r&quot;C:\corpora\penntreebank\parsed\mrg\wsj&quot;</span></span><br><span class="line"><span class="comment"># 使用正则匹配</span></span><br><span class="line">file_pattern = <span class="string">r&quot;.*/wsj_.*\.mrg&quot;</span></span><br><span class="line">ptb = BracketParseCorpusReader(corpus_root, file_pattern)</span><br><span class="line">ptb.fileids()</span><br></pre></td></tr></table></figure>
+<p>双连词运用 <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 依次连续输出最有可能跟在上一个词后面的词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">generate_model</span>(<span class="params">cfdist, word, num=<span class="number">15</span></span>):</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num):</span><br><span class="line">        <span class="built_in">print</span>(word)</span><br><span class="line">        word = cfdist[word].<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 选中语料库，提取文本</span></span><br><span class="line">text = nltk.corpus.genesis.words(<span class="string">&#x27;english-kjv.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 生成双连词</span></span><br><span class="line">bigrams = nltk.bigrams(text)</span><br><span class="line"><span class="comment"># 创建双连词频数对象</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(bigrams)</span><br><span class="line"><span class="comment"># 找出最有可能在`living`后面的单词。</span></span><br><span class="line"><span class="built_in">print</span>(cfd[<span class="string">&#x27;living&#x27;</span>])</span><br><span class="line">generate_model(cfd, <span class="string">&#x27;living&#x27;</span>)</span><br></pre></td></tr></table></figure></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载停用词语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> stopwords</span><br><span class="line"><span class="comment"># 加载英语中的停用词</span></span><br><span class="line">stopwords.words(<span class="string">&#x27;English&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载单词列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> words</span><br><span class="line">words.words()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载人名表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载发音词典</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> cmudict</span><br><span class="line">cmudict.entries()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载比较词表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> swadesh</span><br><span class="line"><span class="comment"># 比较词表的标识码</span></span><br><span class="line">swadesh.fileids()</span><br><span class="line"><span class="comment"># 输出比较词汇中的`en`文本资料</span></span><br><span class="line">swadesh.words(<span class="string">&#x27;en&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载词汇工具</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> toolbox</span><br><span class="line"><span class="comment"># 输出罗托卡特语（Rotokas）的词典文本资料</span></span><br><span class="line">toolbox.entries(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载WordNet</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> wordnet <span class="keyword">as</span> wn</span><br><span class="line"><span class="comment"># 输出`motocar`的同义词</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;motorcar&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义中的同义词集</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemma_names</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义的定义</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).definition</span><br><span class="line"><span class="comment"># 输出同义词集的所有词条</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemmas</span><br><span class="line"><span class="comment"># 输出特定的词条</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出一个词条对应的同义词集</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).synset</span><br><span class="line"><span class="comment"># 输出一个词条的&quot;名字&quot;</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).name</span><br><span class="line"><span class="comment"># 输出所有包含词`car`的词条</span></span><br><span class="line">wn.lemmas(<span class="string">&#x27;car&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取cat词性为名词的意义</span></span><br><span class="line">motorcar = wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 获取motorcar的下位词列表</span></span><br><span class="line">types_of_motorcar = motorcar.hyponyms()</span><br><span class="line"><span class="comment"># 获取motorcar的上位词列表</span></span><br><span class="line">motorcar.hypernyms()</span><br><span class="line"><span class="comment"># 获取motorcar上位词的路径列表</span></span><br><span class="line">paths = motorcar.hypernym_paths()</span><br><span class="line"><span class="comment"># 得到motorcar的根上位（最一般的上位）同义词集</span></span><br><span class="line">motorcar.root_hypernyms()</span><br><span class="line"><span class="comment"># 打开图形化WordNet的浏览器，更加便捷探索关系</span></span><br><span class="line">nltk.app.wordnet()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更多词汇关系</span></span><br><span class="line"><span class="comment"># 整体到部分：想象WordNet网络关系是一棵树</span></span><br><span class="line"><span class="comment"># 得到树的一部分，如树干、树冠等</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).part_meronyms()</span><br><span class="line"><span class="comment"># 得到树的实质，即心材和边材</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).substance_meronyms()</span><br><span class="line"><span class="comment"># 得到许多树构成的集合，即森林</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).member_holonyms()</span><br><span class="line"><span class="comment"># 得到mint所有名词意义的列表</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;mint&#x27;</span>, wn.NOUN)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 想象走路的动作</span></span><br><span class="line"><span class="comment"># 找出走路的一部分动作：抬脚</span></span><br><span class="line">wn.synset(<span class="string">&#x27;walk.v.01&#x27;</span>).entailments()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 反义词</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;supply.n.02.supply&#x27;</span>).antonyms()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用dir()查看词汇关系和同义词集上的定义</span></span><br><span class="line"><span class="built_in">dir</span>(wn.synset(<span class="string">&#x27;harmony.n.02&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 语义相似度</span></span><br><span class="line"><span class="comment"># 初始化一个同义词集中的两个单词</span></span><br><span class="line">right = wn.synset(<span class="string">&#x27;right_whale.n.01&#x27;</span>)</span><br><span class="line">minke = wn.synset(<span class="string">&#x27;minke_whale.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 如果它们有共同的上位词（且层次结构较低），则它们的关系一定十分密切</span></span><br><span class="line">right.lowest_common_hypernyms(minke)</span><br><span class="line"><span class="comment"># 查找同义词集深度量化关系，输出为数值。越小表示离根节点越近</span></span><br><span class="line">wn.synset(<span class="string">&#x27;baleen_whale.n.01&#x27;</span>).min_depth()</span><br><span class="line"><span class="comment"># 两个词 基于上位词层次结构中相互连接的概念之间的最短路径在0-1范围的打分（二者没有关系就返回1）</span></span><br><span class="line">right.path_similarity(minke)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其他相似性度量方法，可以通过help(wn)获取。比如:VerbNet</span></span><br></pre></td></tr></table></figure>
+<p>概念补充：</p>
+<ul>
+<li><p>文本语料库的结构：</p>
+<ul>
+<li><p>孤立的无特别组织的文本集合</p></li>
+<li><p>按文体等分类组织结构</p></li>
+<li><p>分类重叠，主题类别（路透社语料库）</p></li>
+<li><p>随时间变化语言语言用法的改变（就职演说语料库）</p></li>
+</ul></li>
+<li><p>NLTK中定义的基本语料库函数</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-4.png" /></p></li>
+<li><p>条件频率分布</p>
+<p><strong>条件频率分布是频率分布的集合，每个频率分布有一个不同的“条件”。这个条件通常是文本的类别。</strong></p>
+<p>当语料文本被分为几类（文体、主题、作者等）时，我们可以计算每个类别独立的频率分布，研究类别之间的系统性差异。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-5.png" /></p></li>
+<li><p>条件和事件</p>
+<p>对于NLP来说，文本出现的词汇就是事件。条件频率分布需要给每个事件关联一个条件，所以处理的是一个配对序列。每对形式是：（条件，事件）。举例就是，（文本类别，文本中的一个词汇）</p></li>
+<li><p>FreqDist()以一个简单的链表作为输入，ConditionalFreqDist()以一个配对链表作为输入。</p></li>
+<li><p>停用词：
+如<code>the</code>、<code>to</code>这种高频词汇。这种词通常没有什么词汇内容，但是它们又会让文本分析变得困难。所以在需要的情况下，我们就需要从文档中过滤掉他们。</p></li>
+<li><p>WordNet：面向语义的英语词典，类似与传统辞典，但具有更丰富的结构。每个节点对应一个<strong>同义词集</strong>，边表示上/下位词关系（上下级概念与从属概念的关系）</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-6.png" /></p></li>
+<li><p>synset：同义词集，意义想相同的词（或“词条”）的集合。如car.n.01是car的第一个名词的意义，就是同义词集。</p></li>
+<li><p>词条：同义词集和词的配对。</p></li>
 </ul>
-<h3 id="hive常用函数">hive常用函数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以下无特殊说明，返回值皆为string</span></span><br><span class="line"><span class="comment"># 字符串连接</span></span><br><span class="line">hive&gt;concat(strA, strB, ...)</span><br><span class="line"><span class="comment"># 带分隔符的字符串连接，即连接之后字符串存在分隔符，strSEP表示分隔符</span></span><br><span class="line">hive&gt;concat_wd(strSEP, strA, strB, ...)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串截取</span></span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到结尾为止</span></span><br><span class="line">hive&gt;substr(strA, num0)</span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到num1为止</span></span><br><span class="line">hive&gt;substr(strA, num0, num1)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串转大写</span></span><br><span class="line">hive&gt;upper(str)</span><br><span class="line"><span class="comment"># 字符串转小写</span></span><br><span class="line">hive&gt;lower(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 除去字符串中空格</span></span><br><span class="line">hive&gt;trim(str)</span><br><span class="line"><span class="comment"># 除去字符串左边空格</span></span><br><span class="line">hive&gt;ltrim(str)</span><br><span class="line"><span class="comment"># 除去字符串右边空格</span></span><br><span class="line">hive&gt;rtrim(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则替换字符</span></span><br><span class="line"><span class="comment"># 将strA中符合strRE的部分替换为strB</span></span><br><span class="line">hive&gt;regexp_replace(strA, strRE, strB)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重复字符串</span></span><br><span class="line"><span class="comment"># str为字符串，num为重复次数</span></span><br><span class="line">hive&gt;repeat(str, num)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 分割字符串</span></span><br><span class="line"><span class="comment"># 以strpat为分割符，返回字符串数组</span></span><br><span class="line">hive&gt;spilt(str, strpat)</span><br></pre></td></tr></table></figure>
-<h3 id="hive导出数据">hive导出数据</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导出至本地，content表示本地目录</span></span><br><span class="line">hive&gt;insert overwrite <span class="built_in">local</span> directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hdfs，content表示hdfs目录</span></span><br><span class="line">hive&gt;insert overwrite directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hive另一张表othertable，前提hive中存在这张表</span></span><br><span class="line">hive&gt;insert overwrite othertable</span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br></pre></td></tr></table></figure>
-<h3 id="转换类型">转换类型</h3>
-<p>cast(xxx as xx) 将xxx类型转换为xx类型</p>
-<p>下表为是否可转换类型的说明 <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/20170612195815151.png" /></p>
-<p>注：由于表格比较大，这里对一些比较长的字符串进行缩写，ts是timestamp的缩写,bl是boolean的缩写,sl是smallint的缩写,dm是decimal的缩写,vc是varchar的缩写,ba是binary的缩写。</p>
+<p>下一篇：<a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
 ]]></content>
       <categories>
-        <category>Environment</category>
+        <category>NLP</category>
       </categories>
       <tags>
-        <tag>Hadoop</tag>
-        <tag>大数据</tag>
-        <tag>Hive</tag>
+        <tag>Python</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
       </tags>
   </entry>
   <entry>
-    <title>风格迁移转换的两种实现</title>
-    <url>/hexo_blog/2017/08/14/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB%E8%BD%AC%E6%8D%A2%E7%9A%84%E4%B8%A4%E7%A7%8D%E5%AE%9E%E7%8E%B0/</url>
-    <content><![CDATA[<p>本文介绍了如何在Ubuntu16.04环境下搭建caffe和tensorflow，并利用caffe和tensorflow实现了风格转换。同时，介绍了如何训练新的风格模型。</p>
+    <title>NLTK阅读笔记Ⅲ</title>
+    <url>/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+<p>接 <a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
 <span id="more"></span>
-<p><strong>style-transfer</strong></p>
-<p>这个主要是利用caffe+model+code直接实现任意风格的转换。</p>
-<p>最后实现需借助</p>
-<ul>
-<li>一张风格图片</li>
-<li>待转换风格的目标图片</li>
-<li>训练模型</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br></pre></td></tr></table></figure>
-<p>优势：</p>
-<ul>
-<li>可以实现任意风格的转换</li>
-<li>可拓展性强 缺点：</li>
-<li>耗时长（真的很长，如果用CPU的话）</li>
-</ul>
-<p><strong>fast-neural-style-tensorflow</strong></p>
-<p>风格快速迁移转换</p>
-<p>顾名思义，这个比前者能够更快的进行风格转换，速度因机器而异，不过效率比前者确确实实提高了很多倍</p>
-<p>最后实现需借助</p>
-<ul>
-<li>一个训练好的风格模型</li>
-<li>待转换的风格图片</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding</span></span><br><span class="line"> python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br></pre></td></tr></table></figure>
-<p>优势：</p>
-<ul>
-<li>速度相对来说很快</li>
-<li>环境搭建相对来说更容易 缺点：</li>
-<li>可拓展性更弱</li>
-<li>只能转换固定几种风格</li>
-<li>训练新风格模型时间长（比前者生成时间更长）</li>
-</ul>
-<h2 id="style-transfer的实现">style-transfer的实现</h2>
-<p><strong>环境介绍</strong></p>
-<p>Ubuntu16.04 + CPU + python2.7 + caffe</p>
-<h3 id="搭建caffe环境">搭建caffe环境</h3>
-<h4 id="介绍">介绍</h4>
-<p>Caffe是一个清晰而高效的深度学习框架。Caffe是纯粹的C++/CUDA架构，支持命令行、Python和MATLAB接口；可以在CPU和GPU直接无缝切换。</p>
-<p>优势：</p>
-<ul>
-<li>上手快：模型与相应优化都是以文本形式而非代码形式给出。
-Caffe给出了模型的定义、最优化设置以及预训练的权重，方便立即上手。</li>
-<li>速度快：能够运行最棒的模型与海量的数据。
-Caffe与cuDNN结合使用，测试AlexNet模型，在K40上处理每张图片只需要1.17ms.</li>
-<li>模块化：方便扩展到新的任务和设置上。
-可以使用Caffe提供的各层类型来定义自己的模型。</li>
-<li>开放性：公开的代码和参考模型用于再现。</li>
-<li>社区好：可以通过BSD-2参与开发与讨论。</li>
-</ul>
-<h4 id="安装依赖">安装依赖</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># caffe 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler</span><br><span class="line"><span class="built_in">sudo</span> apt-get install --no-install-recommends libboost-all-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libatlas-base-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libhdf5-serial-dev</span><br><span class="line"><span class="comment"># Python 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libgflags-dev libgoogle-glog-dev liblmdb-dev</span><br></pre></td></tr></table></figure>
-<h4
-id="安装caffe可利用virtualenv新建一个纯净环境">安装caffe（可利用virtualenv新建一个纯净环境）</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 我们需要将caffe的源码下载下来，所以需要git</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install git</span><br><span class="line"><span class="comment"># 下载代码</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/BVLC/caffe.git</span><br><span class="line"><span class="comment"># 下载完成后，进入caffe文件夹</span></span><br><span class="line"><span class="built_in">cd</span> caffe</span><br><span class="line"><span class="comment"># 安装caffe的python依赖</span></span><br><span class="line"><span class="keyword">for</span> req <span class="keyword">in</span> $(<span class="built_in">cat</span> requirements.txt); <span class="keyword">do</span> pip install <span class="variable">$req</span>; <span class="keyword">done</span></span><br></pre></td></tr></table></figure>
-<h4 id="配置caffe">配置caffe</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先确保在caffe文件夹中，然后修改配置文件</span></span><br><span class="line"><span class="comment"># 进行备份</span></span><br><span class="line"><span class="built_in">cp</span> Makefile.config.example Makefile.config</span><br><span class="line"><span class="comment"># 进行编辑</span></span><br><span class="line">vim Makefile.config</span><br></pre></td></tr></table></figure>
-<ul>
-<li>因为CPU MODE, 所以在CPU_ONLY := 1前面的#要去掉</li>
-<li>两个路径要改成这样:(添加后面的两个hdf5的路径,
-否则编译时报hdf5错误)</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Whatever else you find you need goes here.</span></span><br><span class="line">INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial</span><br><span class="line">LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial</span><br></pre></td></tr></table></figure>
-<p>修改完成之后，保存退出即可</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行编译</span></span><br><span class="line">make pycaffe</span><br><span class="line">make all</span><br><span class="line">make <span class="built_in">test</span></span><br><span class="line">make runtest</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译结束后，需要设置环境变量</span></span><br><span class="line"><span class="comment"># 首先确保已经进入到caffe文件夹中，之后查看当前路径</span></span><br><span class="line"><span class="built_in">pwd</span></span><br><span class="line"><span class="comment"># 假设输出结果是xxx，那么将xxx复制下来，黏贴至下面xxx的位置</span></span><br><span class="line"><span class="built_in">export</span> PYTHONPATH=xxx/python:<span class="variable">$PYTHONPATH</span></span><br></pre></td></tr></table></figure>
-<h4 id="测试">测试</h4>
-<p>一般来说，如果运气好的话，到这里基本就算安装完成，但我们还是要测试一下</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进入python环境</span></span><br><span class="line">python</span><br><span class="line"><span class="comment"># 导入caffe模块</span></span><br><span class="line">&gt;caffe</span><br><span class="line"><span class="comment"># 如果此处没有报任何错误，恭喜你安装成功</span></span><br></pre></td></tr></table></figure>
-<h4 id="部分报错解决方案">部分报错解决方案</h4>
-<p><strong>最好的方案莫过于，复制报错代码，Google一下 </strong>
-下面提供几种，我安装时的几处坑</p>
-<ul>
-<li>设置环境变量，注意一定是在caffe文件夹下，路径是caffe/python的绝对路径</li>
-<li>安装依赖报错，这个没什么好讲，直接google吧</li>
-<li>make的时候报错，想想看是否编辑了Makefile.config。之后make
-clean一下，再重新编译一下</li>
-<li>CPU_ONLY这个选项为1时，需要修改一下<code>caffe/examples/mnist/lenet_solver.prototxt</code>，将其中的<code>solver_mode:GPU</code>改为CPU</li>
-</ul>
-<p><a
-href="http://blog.csdn.net/sinat_26917383/article/details/53502719">参考文章</a></p>
-<h3 id="实现前准备">实现前准备</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认已安装git，未安装请sudo apt get install git</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/fzliu/style-transfer</span><br><span class="line"></span><br><span class="line"><span class="comment"># pycaffe环境布置</span></span><br><span class="line"><span class="built_in">sudo</span> pip install progressbar</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载训练模型，这里推荐vgg16</span></span><br><span class="line"><span class="comment"># 方法一，使用scripts/ 下的download_models.sh这个方法可能很慢。。。。</span></span><br><span class="line">bash scripts/download_models.sh vgg16</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">wget http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_16_layers.caffemodel</span><br></pre></td></tr></table></figure>
-<p>使用方法二下载的model需要放在./model/vgg16下，使用方法一下载的已经默认放在那</p>
-<h3 id="实现">实现</h3>
-<p>确保在源码文件夹中</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子，model_name直接写vgg16或其他，不需要加具体路径</span></span><br><span class="line">python style.py -s images/style/starry_night.jpg -c images/content/nanjing.jpg -m vgg16 -g -1</span><br></pre></td></tr></table></figure>
-<p>提示：CPU很慢，真的很慢，估计要等一会才会有时间显示出来</p>
-<p>参数解析：</p>
-<ul>
-<li>-s, 风格图位置；</li>
-<li>-c, 内容图位置；</li>
-<li>-m, 模型位置；</li>
-<li>-g, 什么模式，-1为CPU，0为单个GPU，1为两个GPU。</li>
-</ul>
-<h3 id="调整参数">调整参数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保在源码文件夹下,修改style.py配置文件</span></span><br><span class="line">vim style.py</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改文件大小，改为1024</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-l&quot;</span>, <span class="string">&quot;--length&quot;</span>, default=1024, <span class="built_in">type</span>=<span class="built_in">float</span>, required=False, <span class="built_in">help</span>=<span class="string">&quot;maximum image length&quot;</span>)</span><br><span class="line">def transfer_style(self, img_style, img_content, length=1024, ratio=1e5,</span><br><span class="line">n_iter=512, init=<span class="string">&quot;-1&quot;</span>, verbose=False, callback=None)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改迭代次数，个人认为400就差不多了，可自行修改；50次迭代之后改变的就是背景纹理，按需更改</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-n&quot;</span>, <span class="string">&quot;--num-iters&quot;</span>, default=400, <span class="built_in">type</span>=int, required=False, <span class="built_in">help</span>=<span class="string">&quot;L-BFGS iterations&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 还有很多其他可以修改的地方，在此就不一一赘述了</span></span><br></pre></td></tr></table></figure>
-<h2
-id="fast-neural-style-tensorflow的实现">fast-neural-style-tensorflow的实现</h2>
-<p><strong>环境介绍 </strong></p>
-<p>Ubuntu16.04 + CPU + python2.7 + tensorflow 1.0</p>
-<h3 id="搭建tensorflow环境">搭建tensorflow环境</h3>
-<p>环境说明：Ubuntu16.04 注：tensorflow只支持64位系统</p>
-<h4 id="安装前准备">安装前准备</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保有pip工具，一般自带，没有的话请按下面这条命令进行安装</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-pip python-dev</span><br></pre></td></tr></table></figure>
-<h4 id="cpu安装">CPU安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip installl tensorflow</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install tensorflow</span><br></pre></td></tr></table></figure>
-<h4 id="gpu安装">GPU安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装地址请根据自己需求进行声明环境变量</span></span><br><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br></pre></td></tr></table></figure>
-<p><a
-href="https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/1-2-install/">参考文章</a></p>
-<h4 id="源码安装">源码安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载最新的tensorflow项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/tensorflow/tensorflow</span><br><span class="line"><span class="comment"># 进入tensorflow</span></span><br><span class="line"><span class="built_in">cd</span> tensorflow</span><br><span class="line"><span class="comment"># 切换到需要安装的版本分支，以1.0为栗子</span></span><br><span class="line">git checkout r1.0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装bazel</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8&quot;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> /etc/apt/sources.list.d/bazel.list</span><br><span class="line"></span><br><span class="line">curl https://bazel.build/bazel-release.pub.gpg | <span class="built_in">sudo</span> apt-key add -</span><br><span class="line"></span><br><span class="line"><span class="built_in">sudo</span> apt-get update &amp;&amp; <span class="built_in">sudo</span> apt-get install bazel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装依赖</span></span><br><span class="line"><span class="comment"># Python 2</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-numpy python-dev python-pip python-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python 3</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python3-numpy python3-dev python3-pip python3-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行编译，确保在tensorflow文件夹中</span></span><br><span class="line">./configure</span><br><span class="line"><span class="comment"># 提示信息可以一路回车</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译CPU版本</span></span><br><span class="line">bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"><span class="comment"># 编译GPU版本</span></span><br><span class="line">bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译生成的whl包</span></span><br><span class="line">bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装生成的包</span></span><br><span class="line"><span class="built_in">sudo</span> pip install /tmp/tensorflow_pkg/tensorflow*.whl</span><br></pre></td></tr></table></figure>
-<p><a
-href="https://tensorflow.feisky.xyz/install/src.html">参考文章</a></p>
-<h4 id="测试-1">测试</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 打开对应版本的python终端</span></span><br><span class="line">&gt;import tensorflow</span><br><span class="line"><span class="comment"># 未报错即安装成功</span></span><br></pre></td></tr></table></figure>
-<h3 id="实现前准备-1">实现前准备</h3>
-<p>下载已经训练好的模型</p>
-<p><a
-href="https://pan.baidu.com/s/1i4GTS4d#list/path=%2F">百度云盘链接</a></p>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 安装pyyaml</span><br><span class="line">sudo pip install pyyaml</span><br></pre></td></tr></table></figure>
-<h3 id="实现-1">实现</h3>
-<p>确保在源码文件夹中</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子,假设wave.ckpt-done放在了源码文件夹中的models文件夹中</span></span><br><span class="line">python eval.py --model_file models/wave.ckpt-done --image_file img/test.jpg</span><br><span class="line"><span class="comment"># 默认生成位置在generated，默认生成文件名称为res.jpg。可自行在eval.py中更改</span></span><br></pre></td></tr></table></figure>
-<h3 id="训练新的风格模型">训练新的风格模型</h3>
-<p><strong>下载</strong></p>
-<p><a
-href="http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz">vgg16训练模型</a>(500多M)</p>
-<p><a
-href="http://msvocds.blob.core.windows.net/coco2014/train2014.zip">训练数据集</a>(12.6G)</p>
-<p>或者使用wget命令下载</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># vgg16训练模型</span></span><br><span class="line">wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练数据集</span></span><br><span class="line">wget http://msvocds.blob.core.windows.net/coco2014/train2014.zip</span><br></pre></td></tr></table></figure>
-<p><strong>移动模型到对应目录中</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码文件夹中新建文件夹</span></span><br><span class="line"><span class="built_in">mkdir</span> pretrained</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载完成之后需要进行解压</span></span><br><span class="line">tar -zxvf vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 移动文件夹到pretrained</span></span><br><span class="line"><span class="built_in">cp</span> &lt;path of vgg16&gt; &lt;pretrained&gt;</span><br></pre></td></tr></table></figure>
-<p><strong>训练新的模型</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压数据集</span></span><br><span class="line">unzip train2014.zip</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入数据集</span></span><br><span class="line"><span class="built_in">cd</span> train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立软链接，方便训练命令的输入</span></span><br><span class="line"><span class="built_in">ln</span> -s &lt;train2014的绝对路径&gt; train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入源码文件夹中</span></span><br><span class="line"><span class="comment"># 开始训练, wave.yml是作者预先配置好的文件，如果训练自己的模型，需要自己新写一份yml文件</span></span><br><span class="line">python train.py -c conf/wave.yml</span><br></pre></td></tr></table></figure>
-<h2 id="总结">总结</h2>
-<p>如果只是单纯的想玩玩风格转换的话，可以试试fast-neural-style-tensorflow，不过训练新模型还是不建议（特殊需要除外），耗时耗力</p>
-<p>如果想锻炼自己，从零开始学习，可以从读论文开始，然后试试style-transfer代码</p>
-<p><a
-href="http://blog.csdn.net/sinat_26917383/article/details/53978519">style-transfer
-参考文章</a></p>
-<p><a href="https://github.com/fzliu/style-transfer">style-transfer
-代码</a></p>
-<p><a
-href="https://github.com/hzy46/fast-neural-style-tensorflow/blob/master/README.md#use-trained-models">fast-neural-style-tensorflow
-代码</a></p>
+<h2 id="第六章学习分类文本">第六章：学习分类文本</h2>
+<p>目标：</p>
+<ol type="1">
+<li>找语言数据中的分类特征</li>
+<li>构建模型</li>
+<li>从模型中学习知识</li>
+</ol>
+<p>分类学习，一般分为有监督学习、无监督学习、半监督学习。</p>
+<p>分类的定义：将给定的输入选择正确（合适）的类标签。这里值得注意的是，输入之间是相互独立的，标签集是预先定义好的。延伸的任务，每个实例（输入）可以划分为多个标签；分类问题不一定预先定义标签集，类似K-Means聚类；序列分类中，一个输入链表可以作为一个整体分类。</p>
+<p>在这里，我们预先得到一些包含正确标签的语料（实例/输入），根据这些数据建立得到的模型进行分类，我们称为有监督分类。有监督分类中，在训练过程中，需要将每个一个输入值转换为特征集（特征提取器）。特征集对应着输出类标签。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，提取输入字符串中最后一个字母。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="comment"># 返回特征集</span></span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;last_letter&#x27;</span>: word[-<span class="number">1</span>]&#125;</span><br><span class="line"><span class="comment"># 导入姓名语料库, 获取男女姓名列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 随机打乱男女姓名列表</span></span><br><span class="line">random.shuffle(names)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 划分训练集和测试集</span></span><br><span class="line">featuresets = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 利用贝叶斯分类来训练模型</span></span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行预测 </span></span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Neo&#x27;</span>))</span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Trinity&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 准确率</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最有效的五个特征（字母）</span></span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string">             last_letter = &#x27;k&#x27;              male : female =     43.4 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;a&#x27;            female : male   =     32.9 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;f&#x27;              male : female =     16.1 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;p&#x27;              male : female =     12.6 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;v&#x27;              male : female =     10.6 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span>                 </span><br><span class="line"><span class="comment"># 输出的比率称为 似然比，可以用于比较不同特征-结果关系</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 处理大型语料库时，构建一个包含每一个实例的特征的单独的链表会使用大量的内存。这种情况下，可以使用下面的函数，使返回一个行为像一个链表而不会在内存存储所有特征集的对象</span></span><br><span class="line"><span class="keyword">from</span> nltk.classify <span class="keyword">import</span> apply_features</span><br><span class="line">train_set = apply_features(gender_features, names[<span class="number">500</span>:])</span><br><span class="line">test_set = apply_features(gender_features, names[:<span class="number">500</span>])</span><br></pre></td></tr></table></figure>
+<h3 id="选取特征">选取特征</h3>
+<p>过拟合：用于一个给定的学习算法的特征的数目是有限的——如果提供太多的特征，那么该算法将高度依赖训练数据的特性而一般化到新的例子的效果不会很好。</p>
+<p>过拟合当运作在小训练集上时尤其会有问题。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，过拟合提取特征。这里请对比gender_features2与上一版本的区别</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features2</span>(<span class="params">name</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&quot;firstletter&quot;</span>] = name[<span class="number">0</span>].lower()</span><br><span class="line">    features[<span class="string">&quot;lastletter&quot;</span>] = name[–<span class="number">1</span>].lower()</span><br><span class="line">    <span class="keyword">for</span> letter <span class="keyword">in</span> <span class="string">&#x27;abcdefghijklmnopqrstuvwxyz&#x27;</span>:</span><br><span class="line">        features[<span class="string">&quot;count(%s)&quot;</span> % letter] = name.lower().count(letter)</span><br><span class="line">        features[<span class="string">&quot;has(%s)&quot;</span> % letter] = (letter <span class="keyword">in</span> name.lower())</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">featuresets = [(gender_features2(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 对比上一版本的预测结果，基本可以发现精确度下降了</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<p>一旦初始特征集被选定，完善特征集的一个非常有成效的方法是<strong>错误分析</strong>。首先，我们选择一个
+开发集，包含用于创建模型的语料数据。然后将这种开发集分为 训练集和
+开发测试集。训练集用于训练模型，开发测试集用于进行错误分析，测试集用于系统的最终评估。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 划分训练集、开发集、开发测试集</span></span><br><span class="line">train_names = names[<span class="number">1500</span>:]</span><br><span class="line">devtest_names = names[<span class="number">500</span>:<span class="number">1500</span>]</span><br><span class="line">test_names = names[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 错误分析</span></span><br><span class="line">errors = []</span><br><span class="line"><span class="keyword">for</span> (name, tag) <span class="keyword">in</span> devtest_names:</span><br><span class="line">    guess = classifier.classify(gender_features(name))</span><br><span class="line">    <span class="keyword">if</span> guess != tag:</span><br><span class="line">        errors.append((tag, guess, name))</span><br><span class="line">      </span><br><span class="line"><span class="comment"># 输出错误分类列表中，猜错的类别及名字</span></span><br><span class="line"><span class="keyword">for</span> (tag, guess, name) <span class="keyword">in</span> <span class="built_in">sorted</span>(errors):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;correct=%-8s guess=%-8s name=%-30s&#x27;</span> % (tag, guess, name))</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 观察输出进行分析，得到一些结果。</span></span><br><span class="line"><span class="comment"># yn结尾的名字显示以女性为主，虽然n结尾的名字往往是男性</span></span><br><span class="line"><span class="comment"># 以ch结尾的名字通常是男性，虽然h结尾的名字倾向于是女性</span></span><br><span class="line"><span class="comment"># 根据结论，调整特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;suffix1&#x27;</span>: word[-<span class="number">1</span>:], <span class="string">&#x27;suffix2&#x27;</span>: word[-<span class="number">2</span>:]&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"><span class="comment"># 最终结果理论上会进行提高</span></span><br></pre></td></tr></table></figure>
+<p>不断地进行错误分析改善特征提取器，可以不断提高预测的准确度。每次错误分析应该选取不同的开发测试/训练进行分割，以检查新改进分类器可能产生的新的错误模式。</p>
+<h3 id="文档分类">文档分类</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> movie_reviews</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据电影评论分类（&#x27;neg&#x27;, &#x27;pos&#x27;）获取电影评论</span></span><br><span class="line">documents = [(<span class="built_in">list</span>(movie_reviews.words(fileid)), category) <span class="keyword">for</span> category <span class="keyword">in</span> movie_reviews.categories() <span class="keyword">for</span> fileid <span class="keyword">in</span> movie_reviews.fileids(category)]</span><br><span class="line"><span class="comment"># 随机打乱数据</span></span><br><span class="line">random.shuffle(documents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立文档分类的特征提取器</span></span><br><span class="line"><span class="comment"># 使用FreqDist会比一般的list更快</span></span><br><span class="line">all_words = nltk.FreqDist(w.lower() <span class="keyword">for</span> w <span class="keyword">in</span> movie_reviews.words())</span><br><span class="line">word_features = <span class="built_in">list</span>(all_words.keys())[:<span class="number">2000</span>]</span><br><span class="line"><span class="comment"># 每个词是否在一个给定的文档中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">document_features</span>(<span class="params">document</span>):</span><br><span class="line">    document_words = <span class="built_in">set</span>(document)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> word_features:</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word] = (word <span class="keyword">in</span> document_words)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练文档分类模型</span></span><br><span class="line">featuresets = [(document_features(d), c) <span class="keyword">for</span> (d,c) <span class="keyword">in</span> documents]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">100</span>:], featuresets[:<span class="number">100</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 检验模型</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>) </span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string"> contains(unimaginative) = True              neg : pos    =      7.7 : 1.0</span></span><br><span class="line"><span class="string">    contains(schumacher) = True              neg : pos    =      7.5 : 1.0</span></span><br><span class="line"><span class="string">        contains(suvari) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">          contains(mena) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">        contains(shoddy) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="comment"># 显然可以发现，`unimaginative`这个词是负面的概率是正面的7.7倍</span></span><br></pre></td></tr></table></figure>
+<h3 id="词性标注">词性标注</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为词选择词性标记，通过词的后缀来选择</span></span><br><span class="line">suffix_fdist = nltk.FreqDist()</span><br><span class="line"><span class="keyword">for</span> word <span class="keyword">in</span> brown.words():</span><br><span class="line">    word = word.lower()</span><br><span class="line">    suffix_fdist[word[-<span class="number">1</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">2</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">3</span>:]] += <span class="number">1</span></span><br><span class="line">common_suffixes = <span class="built_in">list</span>(suffix_fdist.keys())[:<span class="number">100</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 现在通过特征提取器，计算哪个后缀最有信息量（最具代表性)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">word</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> suffix <span class="keyword">in</span> common_suffixes:</span><br><span class="line">        features[<span class="string">&#x27;endswith(%s)&#x27;</span> % suffix] = word.lower().endswith(suffix)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">tagged_words = brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 利用特征提取器选取特征</span></span><br><span class="line">featuresets = [(pos_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> tagged_words]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line"><span class="comment"># 利用决策树方法来训练模型</span></span><br><span class="line">classifier = nltk.DecisionTreeClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.classify(pos_features(<span class="string">&#x27;cats&#x27;</span>))</span><br><span class="line"><span class="comment"># 将nltk运算过程，以伪代码方式输出。设定决策树深度为4</span></span><br><span class="line">classifier.pseudocode(depth=<span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据上下文语境，更新特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">pos_features(brown.sents()[<span class="number">0</span>], <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 检验新的词性标注器性能</span></span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">featuresets = []</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> tagged_sents:</span><br><span class="line">    untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">    <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">        featuresets.append((pos_features(untagged_sent, i), tag))</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<p>以上是在结合整个词集做的标注器，但是往往针对某些特殊情况，如形容词后面有很大概率上是名词。对于这种情况，我们的词性标注器目前还没有进行独立对待。</p>
+<h3 id="序列分类">序列分类</h3>
+<p>联合分类器：捕捉相关的分类任务之间的依赖关系，收集有关输入，选择适当的标签</p>
+<p>连续分类（贪婪序列分类）：为第一个输入找到最有可能的类标签，然后使用这个问题的答案帮助找到下一个输入的最佳的标签。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i, history</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]&#125;</span><br><span class="line">	<span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = history[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="keyword">class</span> <span class="title class_">ConsecutivePosTagger</span>(nltk.TaggerI):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, train_sents</span>):</span><br><span class="line">        train_set = []</span><br><span class="line">        <span class="keyword">for</span> tagged_sent <span class="keyword">in</span> train_sents:</span><br><span class="line">            untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">            history = []</span><br><span class="line">            <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">                featureset = pos_features(untagged_sent, i, history)</span><br><span class="line">                train_set.append( (featureset, tag) )</span><br><span class="line">                history.append(tag)</span><br><span class="line">        <span class="variable language_">self</span>.classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tag</span>(<span class="params">self, sentence</span>):</span><br><span class="line">        history = []</span><br><span class="line">        <span class="keyword">for</span> i, word <span class="keyword">in</span> <span class="built_in">enumerate</span>(sentence):</span><br><span class="line">            featureset = pos_features(sentence, i, history)</span><br><span class="line">            tag = <span class="variable language_">self</span>.classifier.classify(featureset)</span><br><span class="line">            history.append(tag)</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">zip</span>(sentence, history)</span><br><span class="line">  </span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(tagged_sents) * <span class="number">0.1</span>)</span><br><span class="line">train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]</span><br><span class="line">tagger = ConsecutivePosTagger(train_sents)</span><br><span class="line">tagger.evaluate(test_sents)</span><br></pre></td></tr></table></figure>
+<h3 id="其他序列模型">其他序列模型</h3>
+<p>以上模型有一个致命问题就是无法修复已经标注错误的词性。另一种方案<strong>隐马尔科夫模型</strong>可以为所有的可能序列进行打分，选择得分最高的序列。这种模型的缺点就是计算量相当大，可以采用动态规划来解决这种问题。基于这种模型，产生了<strong>最大熵马尔可夫模型</strong>和<strong>线性链条件随机场模型</strong>，二者为可能序列打分的算法不同。</p>
+<h3 id="句子分割">句子分割</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sents = nltk.corpus.treebank_raw.sents()</span><br><span class="line"><span class="comment"># 单独的句子标识符的合并链表</span></span><br><span class="line">tokens = []</span><br><span class="line"><span class="comment"># 所有句子边界标识索引</span></span><br><span class="line">boundaries = <span class="built_in">set</span>()</span><br><span class="line">offset = <span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> sent <span class="keyword">in</span> sents:</span><br><span class="line">    tokens.extend(sent)</span><br><span class="line">    offset += <span class="built_in">len</span>(sent)</span><br><span class="line">    boundaries.add(offset-<span class="number">1</span>)</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">punct_features</span>(<span class="params">tokens, i</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;next-word-capitalized&#x27;</span>: tokens[i+<span class="number">1</span>][<span class="number">0</span>].isupper(),</span><br><span class="line">            <span class="string">&#x27;prevword&#x27;</span>: tokens[i-<span class="number">1</span>].lower(),</span><br><span class="line">            <span class="string">&#x27;punct&#x27;</span>: tokens[i],</span><br><span class="line">            <span class="string">&#x27;prev-word-is-one-char&#x27;</span>: <span class="built_in">len</span>(tokens[i-<span class="number">1</span>]) == <span class="number">1</span>&#125;</span><br><span class="line"></span><br><span class="line">featuresets = [(punct_features(tokens, i), (i <span class="keyword">in</span> boundaries))</span><br><span class="line">				<span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="built_in">len</span>(tokens)-<span class="number">1</span>)</span><br><span class="line">				<span class="keyword">if</span> tokens[i] <span class="keyword">in</span> <span class="string">&#x27;.?!&#x27;</span>]</span><br><span class="line"><span class="comment"># 进行训练，检验模型准确率</span></span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<h3 id="识别对话行为类型">识别对话行为类型</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取即时消息会话语料库</span></span><br><span class="line">posts = nltk.corpus.nps_chat.xml_posts()[:<span class="number">10000</span>]</span><br><span class="line"><span class="comment"># 特征提取器，每个帖子包含什么词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">dialogue_act_features</span>(<span class="params">post</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> nltk.word_tokenize(post):</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word.lower()] = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="comment"># 构建模型，判断是否为对话行为</span></span><br><span class="line">featuresets = [(dialogue_act_features(post.text), post.get(<span class="string">&#x27;class&#x27;</span>)) <span class="keyword">for</span> post <span class="keyword">in</span> posts]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<h3 id="识别文字蕴含">识别文字蕴含</h3>
+<p>识别文字蕴含（Recognizing textual entailment，RTE）是判断文本 T
+的一个给定片段是否蕴含着另一个叫做“假设”的文本。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">rte_features</span>(<span class="params">rtepair</span>):</span><br><span class="line">    <span class="comment">#  RTEFeatureExtractor 类建立了一个除去一些停用词后在文本和假设中都有的词汇包，然后计算重叠和差异</span></span><br><span class="line">    extractor = nltk.RTEFeatureExtractor(rtepair)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&#x27;word_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;word_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 原文中rte识别率只是刚刚超过了58%，所以不多做介绍</span></span><br><span class="line">nltk.classify.rte_classify </span><br></pre></td></tr></table></figure>
+<h3 id="涉及到的定义">涉及到的定义</h3>
+<p>文中介绍一些关于评估模型、决策树、朴素贝叶斯分类器的知识。由于笔者已掌握这些基础模型知识所以在这里略去，以下介绍一些基本的定义。</p>
+<p>测试集：选择适当的比例，防止过拟合</p>
+<p>准确度：测量测试集上分类器正确标注的输入的比例。</p>
+<p>精确度（Precision），表示我们发现的项目中有多少是相关的，TP/(TP+
+FP)。</p>
+<p>召回率（Recall），表示相关的项目中我们发现了多少，TP/(TP+ FN)。</p>
+<p>F- 度量值（F-Measure）（或
+F-得分，F-Score），组合精确度和召回率为一个单独的得分，被定义为精确度和召回率的调和平均数(2
+× Precision × Recall)/(Precision+Recall)。</p>
+<p>混淆矩阵：其中每个 cells[i,j]表示正确的标签 i 被预测为标签 j 的次数
+。因此，对角线项目（即
+cells[i,i]）表示正确预测的标签，非对角线项目表示错误。nltk中可以使用
+<code>nltk.ConfusionMatrix</code>函数</p>
+<p>交叉验证：在不同的测试集上执行多个评估，然后组合这些评估的得分</p>
+<p>信息增益：用给定的特征分割输入值，衡量数据变得更有序的程度</p>
+<p>朴素贝叶斯假设（独立性假设）：每个输入值是通过首先为那个输入值选择一个类标签，然后产生每个特征的方式产生的
+，每个特征与其他特征完全独立。</p>
+<p>零计数：如果训练集中有特征从来没有和给定标签一起出现，导致给定标签的标签可能性为
+0。</p>
+<p>平滑：给定一个初值，解决零计数问题。<code>nltk.probability</code>提供了多种平滑技术</p>
+<p>Heldout估计使用一个heldout
+语料库计算特征频率与特征概率之间的关系。</p>
+<p>非二元特征：如果标签是1,2,4,5这种多元标签，可以转换0&lt;x&lt;3,
+3&lt;x&lt;6这种二元标签</p>
+<p>双重计数：在训练过程中特征的贡献被分开计算，但当使用分类器为新输入选择标签时，这些特征的贡献被组合。解决方案就是为每一个特征贡献设定一个权重。</p>
+<p>最大熵分类器：与朴素贝叶斯类似，使用搜索技术找出一组将最大限度地提高分类器性能的参数。避免使用广义
+迭代缩放（Generalized Iterative Scaling，GIS）或改进的迭代缩放（Improved
+Iterative Scaling，IIS），这两者都比共轭梯度（Conjugate Gradient，CG
+）和 BFGS 优化方法慢很多</p>
+<p>联合特征：每个接收它自己的参数的标签和特征的组合。联合特征是有标签的的值的属性，而（简单）特征是未加标签的值的属性。</p>
+<p>一般情况下，
+最大熵原理是说在与我们所知道的一致的的分布中，我们会选择熵最高（分布最均匀）的。</p>
+<p>对于每个联合特征，最大熵模型计算该特征的“经验频率”——即它出现在训练集中的频率。然后，它搜索熵最大的分布，同时也预测每个联合特征正确的频率。</p>
+<p>一般情况下，生成式模型确实比条件式模型强大。但是前者所需的参数会大于后者。</p>
 ]]></content>
       <categories>
-        <category>Environment</category>
+        <category>NLP</category>
       </categories>
       <tags>
         <tag>Python</tag>
-        <tag>Tensorflow</tag>
-        <tag>Caffe</tag>
-        <tag>风格迁移</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
       </tags>
   </entry>
   <entry>
-    <title>crontab命令</title>
-    <url>/hexo_blog/2017/08/24/%E8%BF%90%E7%BB%B4/crontab%E5%91%BD%E4%BB%A4/</url>
-    <content><![CDATA[<p>学会使用crontab定时任务，提高工作效率。详解了crontab的基本格式和例子，以及相关目录介绍和日志文件的设置。</p>
+    <title>神经网络笔记Ⅰ</title>
+    <url>/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
+    <content><![CDATA[<p>本篇文章只做一个神经网络入门知识的梳理和个人的理解。</p>
 <span id="more"></span>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看当前用户的定时任务，也可以使用crontab -uroot -l查看指定用户的定时任务</span></span><br><span class="line">crontab -l</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编辑crontab定时任务，貌似是使用vim的编辑命令</span></span><br><span class="line">crontab -e</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其实基本上知道这两个就可以处理定时任务了</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除当前用户的所有定时任务，不知道为什么会有这种设定（手贱按下过一次，恢复起来真的惨）</span></span><br><span class="line">crontab -r</span><br><span class="line"><span class="comment"># 如果按到了，就去日志文件里面找记录，尽可能恢复命令吧</span></span><br></pre></td></tr></table></figure>
-<p>下面讲讲编辑任务的格式</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 基本格式</span></span><br><span class="line"></span><br><span class="line">  *  *  *  *  * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 分 时 日 月 周 需要执行的命令</span></span><br></pre></td></tr></table></figure>
-<p>可视化测试在线网站，https://crontab.guru/</p>
-<p>几个栗子</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每分钟执行一次command命令，空格表示间隔</span></span><br><span class="line">* * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时执行一次，‘/’表示频率</span></span><br><span class="line">0 */1 * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时的10分和40分各执行一次，‘,’表示并列</span></span><br><span class="line">10,40 * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每个星期一的10-14点的10，40执行，‘-’表述区间</span></span><br><span class="line">10,40 10-14 * * 1 <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每天的8，13，20点执行，注意此时表示分的那个*要替换为0</span></span><br><span class="line">0 8,13,20 * * * <span class="built_in">command</span></span><br></pre></td></tr></table></figure>
-<p>日志文件一般存放在
-<code>/var/log/cron*</code>中，不过我的Ubuntu好像没有开启cron日志，可以在
-<code>/etc/rsyslog.d/50-default.conf</code>中进行设置（Centos在
-<code>/etc/rsyslog.conf</code>中设置）</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/rsyslog.d/50-default.conf</span><br><span class="line"><span class="comment"># 找到cron*, 去掉前面的#注释即可</span></span><br><span class="line"><span class="comment"># 重启rsyslog</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service rsyslog  restart</span><br></pre></td></tr></table></figure>
-<p>crontab相关目录介绍</p>
-<p><code>/etc/anacrontab</code>:
-这个文件存着系统级的任务。它主要用来运行每日的(daily)，每周的(weekly),每月的(monthly)的任务。一般不在此文件安装自己的任务</p>
-<p><code>/etc/cron.d/</code>: 此目录下存放的是系统级任务的任务文件。</p>
-<p><code>/var/spool/cron/</code>:
-此目录下存放各个用户的任务文件。各个用户的任务存放在以自已用户名为文件名的任务文件中。此文件中的指令行没有用户域。</p>
-<p><code>/etc/crontab</code>:
-crontab的主要配置文件，执行了每小时、每日、每周、每月的脚本文件</p>
-<p><code>/etc/cron.hourly</code>、<code>/etc/cron.daily</code>、<code>/etc/cron.weekly</code>、<code>/etc/cron.monthly</code>：这四个目录下分别对应着每隔一段时间需要执行的脚本任务，脚本使用shell命令编写</p>
-<p><a href="https://segmentfault.com/a/1190000002628040"></a></p>
+<p>关于神经网络的一些基本概念在这里就不做介绍，首先我们进行一些概念和符号上的约定。</p>
+<p>假设一个矩阵如下：</p>
+<p><span class="math display">\[
+\mathbf{X} = \left[ \begin{array}{ccc} 1 &amp; 2 &amp; 3 \\ 2 &amp; 3
+&amp; 1 \\ 3 &amp; 2 &amp; 1  \end{array}\right]
+\]</span></p>
+<p><strong>点乘</strong>： <span
+class="math inline">\(\cdot\)</span></p>
+<p><span class="math display">\[
+\mathbf{X} \cdot \mathbf{X} = \left [ \begin{array}{ccc} 1*1 &amp; 2* 2
+&amp; 3* 3 \\ 2*2 &amp; 3*3 &amp; 1*1 \\ 3*3&amp; 2*2 &amp; 1*1
+\end{array} \right]= \left [ \begin{array}{ccc} 1 &amp; 4 &amp; 9 \\
+4&amp; 9&amp; 1 \\ 9&amp; 4&amp; 1 \end{array} \right]
+\]</span></p>
+<p><strong>叉乘</strong>： *</p>
+<p><span class="math display">\[
+\mathbf{X} * \mathbf{X} = \left [ \begin{array}{ccc} 1* 1 + 2*2+
+3*3&amp; 1*2+2*3+3*2 &amp; 1*3+2*1+3*1 \\ 2*1+3*2+1*3 &amp; 2*2+3*3+1*2
+&amp; 2*3+3*1+1*1 \\ 3*1+2*2+1*3&amp; 3*2+2*3+1*2&amp;
+3*3+2*1+1*1\end{array} \right]= \left [ \begin{array}{ccc}14 &amp; 14
+&amp; 8 \\11&amp; 15&amp; 10 \\ 10&amp; 14&amp; 12 \end{array} \right]
+\]</span></p>
+<p><strong>已知数据</strong>：
+矩阵<strong>X</strong>为训练数据集，矩阵<strong>y</strong>为训练数据集的标签</p>
+<h2 id="最简单的神经网络">最简单的神经网络</h2>
 <p><a
-href="http://www.cnblogs.com/peida/archive/2013/01/08/2850483.html">每天一个linux命令（50）：crontab命令</a></p>
-<p><a href="https://segmentfault.com/a/1190000002628040">Linux
-下执行定时任务 crontab 命令详解</a></p>
+href="https://juejin.im/entry/5ab495f151882521d657806f">基于Numpy实现神经网络：反向传播</a></p>
+<p>最简单的神经网络就是只有输入层和输出层的网络结构（双层神经网络）。</p>
+<p>关于激活函数简单介绍：</p>
+<p>如果不使用激活函数，那么就与多层感知机（MLP）相当。引入之后，下一层的输入就不再是线性组合，输出就有意义。</p>
+<p>如果不使用激活函数，那么输入与输出都是线性变换，无法做到非线性分类。</p>
+<h3 id="激活函数">激活函数</h3>
+<p>这是公式，值域为[0,1]，它的导数自证。</p>
+<p>作用：</p>
+<ol type="1">
+<li>引入非线性因素</li>
+<li>线性变换</li>
+<li>激活函数，并不是去激活什么，而是指如何把“激活的神经元的特征”通过函数把特征保留并映射出来（保留特征，去除一些数据中是的冗余。激励就是样本的特征值），这是神经网络能解决非线性问题关键。</li>
+</ol>
+<p><code>sigmoid</code>的导数，即使用了 <code>deriv=True</code>。</p>
+<p>作用：用它的输出创建它的导数，降低高信度预测的错误</p>
+<p>注意：如果神经网络提前达到预期的结果（损失函数小于一定的范围），可以提前终止</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="comment"># 矩阵计算</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="comment"># 生成随机数</span></span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="comment"># 定义激活函数, 这里用sigmoid函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># 如果满足条件就返回sigmoid的导数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="comment"># 不满足直接返回sigmoid函数</span></span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定训练数据, X规模为4x3, y规模为4x1</span></span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 初始化权重数据, 用随机数进行初始化，矩阵规模3x1</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行迭代</span></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 输入层为训练数据</span></span><br><span class="line">    l0 = X</span><br><span class="line">    <span class="comment"># np.dot表示叉乘, 进行激活操作， 输出层</span></span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    <span class="comment"># 计算偏差</span></span><br><span class="line">    l1_error = y - l1</span><br><span class="line">    <span class="comment"># 误（偏）差加权导数。偏差乘以sigmoid在l1处的斜率，用以更新权重</span></span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练完成，输出l1看看</span></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
+<h2 id="三层神经网络">三层神经网络</h2>
+<p>增加了一层网络，在输入层和输出层中间。这里做一个规定，在输入层和输出层之间的网络结构我们称为隐藏层。</p>
+<p><strong>一层隐层网络就是一层特征层次，每一个神经元可以类似看作一个特征属性</strong></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 初始化权重数据, 用随机数进行初始化， 这里的规模同X、y矩阵的规模</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">4</span>)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((<span class="number">4</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    l0 = X</span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    <span class="comment"># 加了一层神经网络，称为隐藏层</span></span><br><span class="line">    l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">    l2_error = y - l2</span><br><span class="line">    l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">    <span class="comment"># 计算完l2的误差之后，再用结果去计算l1的误差</span></span><br><span class="line">    l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_1 += np.dot(l1.T, l2_delta)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
+<h2 id="改进神经网络v1.0增加梯度">改进神经网络v1.0——增加梯度</h2>
+<p><a
+href="http://iamtrask.github.io/2015/07/27/python-network-part2/">A
+Neural Network in 13 lines of Python (Part 2 - Gradient Descent)</a></p>
+<p>关于梯度的定义请自行了解。</p>
+<p>增加了梯度可以更快的找到较为理想的结果。</p>
+<p>关于梯度上升有下面几种方法（梯度下降同理）：</p>
+<ol type="1">
+<li><p>原始的梯度上升</p>
+<p>目标：找到某个函数的最大值。每次沿函数的梯度方向探寻。一直进行迭代，直到到达某个停止条件（迭代次数限制或某个误差范围）</p></li>
+<li><p>随机梯度上升(SGD)</p>
+<p>不同点在于，第一种方法每次是遍历所有的数据集（一百以内的数据集可以接受用上面的方法）。而随机梯度则是只使用一个样本点来更新回归系数。</p></li>
+<li><p>改进版随机梯度上升</p>
+<p>用随机的一个样本来更新回归系数。</p></li>
+<li><p>批梯度上升</p>
+<p>切分样本集，随机取出切分后的某些样本，进行遍历</p></li>
+</ol>
+<p>损失函数（代价函数）</p>
+<ol type="1">
+<li><p>二次代价函数</p>
+<p><span class="math display">\[
+C= \frac{1}{2n}\sum_\limits{n}[y(x)-a^L(x)]^2
+\]</span></p></li>
+<li><p>交叉熵代价函数</p>
+<p><span class="math display">\[
+C=\frac{1}{2n}\sum_\limits{x}[y\ln a-(1-y)\ln(1-a)]
+\]</span></p></li>
+<li><p>对数似然函数</p>
+<p><span class="math display">\[
+C=\frac{1}{n}\sum_\limits{x}\sum_\limits{k}y_k\log a_k
+\]</span></p>
+<p>前向传播是函数求偏导，反向传播减少了这个计算量</p></li>
+</ol>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, <span class="number">4</span>)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((<span class="number">4</span>, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 加入步长</span></span><br><span class="line">alphas = [<span class="number">0.001</span>,<span class="number">0.01</span>,<span class="number">0.1</span>,<span class="number">1</span>,<span class="number">10</span>,<span class="number">100</span>,<span class="number">1000</span>]</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> alpha <span class="keyword">in</span> alphas:</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">        l0 = X</span><br><span class="line">        l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">        <span class="comment"># 加了一层神经网络，称为隐藏层</span></span><br><span class="line">        l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">        l2_error = y - l2</span><br><span class="line">        l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">        <span class="comment"># 计算完l2的误差之后，再用结果去计算l1的误差</span></span><br><span class="line">        l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">        l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">        <span class="comment"># 更新权重的时候加入步长</span></span><br><span class="line">        syn_1 += alpha * np.dot(l1.T, l2_delta)</span><br><span class="line">        syn_0 += alpha * np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;alpha:&quot;</span>, alpha, <span class="string">&quot;l1:&quot;</span>, l1)</span><br></pre></td></tr></table></figure>
+<h2
+id="改进神经网络v2.0更改隐单元个数">改进神经网络v2.0——更改隐单元个数</h2>
+<p>更改网络的神经元个数，这个可能就是所谓的玄学调参</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line">X = np.array([[<span class="number">0</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">0</span>,<span class="number">1</span>],[<span class="number">1</span>,<span class="number">1</span>,<span class="number">1</span>]])</span><br><span class="line">y = np.array([[<span class="number">0</span>,<span class="number">1</span>,<span class="number">1</span>,<span class="number">0</span>]]).T</span><br><span class="line"></span><br><span class="line"><span class="comment"># 定义神经元个数，随机</span></span><br><span class="line">hidden_dim = np.random.randint(<span class="number">2</span>, <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((<span class="number">3</span>, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, <span class="number">1</span>)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    l0 = X</span><br><span class="line">    l1 = sigmoid(np.dot(l0, syn_0))</span><br><span class="line">    l2 = sigmoid(np.dot(l1, syn_1))</span><br><span class="line">    l2_error = y - l2</span><br><span class="line">    l2_delta = l2_error * sigmoid(l2, deriv=<span class="literal">True</span>)</span><br><span class="line">    l1_error = np.dot(l2_delta, syn_1.T)</span><br><span class="line">    l1_delta = l1_error * sigmoid(l1, deriv=<span class="literal">True</span>)</span><br><span class="line">    syn_1 += np.dot(l1.T, l2_delta)</span><br><span class="line">    syn_0 += np.dot(l0.T, l1_delta)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(l1)</span><br></pre></td></tr></table></figure>
+<h2 id="改进神经网络v3.0dropout">改进神经网络v3.0——dropout</h2>
+<p><a href="http://iamtrask.github.io/2015/07/28/dropout/">Hinton's
+Dropout in 3 Lines of Python</a></p>
+<p>增加<strong>dropout_percent</strong>。目的是为了防止过拟合，一种正则化的手段。在迭代的时候，对某一（几）层神经网络进行<strong>drop_out</strong>。这里举例用二项分布采样的方法进行。<code>np.random.binomial(n, p, size=None)</code>。</p>
+<p>Dropout是指在模型训练时<strong>随机</strong>让网络<strong>某些隐含层节点</strong>的权重不工作，不工作的那些节点可以暂时认为不是网络结构的一部分，但是它的权重得保留下来（只是暂时不更新而已）。</p>
+<p>一些理由解释：</p>
+<ol type="1">
+<li>权值的更新不依赖于一些固定关系隐含节点的共同作用。阻止了某些特征仅仅在其它特定特征下才有效果的情况。即随机选取当前隐含层的部分节点</li>
+<li>模型平均概念（选取每次计算过程的相对最优解），使用了
+<code>dropout</code>之后每次的网络结构不同，样本不同对应的输出模型也不同。（个人感觉有点像加入随机）</li>
+<li>生物进化解释，不断适应变化情况，有效阻止过拟合（避免环境改变物种面临死亡）</li>
+<li>朴素贝叶斯（native
+bayes）属于dropout的一种特例：各个特征之间相互独立。在训练样本少的情况下，单独对每个特征进行学习（dropout是训练一部分特征）。</li>
+</ol>
+<p>这里假设对<span class="math inline">\(l_1\)</span>进行drop_out。</p>
+<p>定义 <code>len(x)=3, hidden_dim=4, dropout_percent=0.2</code></p>
+<p>代入数值计算,</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">np.random.binomial([array([[ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>],</span><br><span class="line">                           [ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>],</span><br><span class="line">                           [ <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>,  <span class="number">1.</span>]])], <span class="number">0.8</span>)[<span class="number">0</span>] * (<span class="number">1</span>/<span class="number">0.8</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认的dropout_percent是0.5, 即如果不加如dropout_percent的话</span></span><br><span class="line"><span class="comment"># 如果是对图像进行操作的话，建议不超过0.25</span></span><br><span class="line">dropout_percent = <span class="number">0.2</span></span><br><span class="line"></span><br><span class="line">...</span><br><span class="line">l1 *= np.random.binomial([np.ones((<span class="built_in">len</span>(X), hidden_dim))],<span class="number">1</span> - dropout_percent)[<span class="number">0</span>] *(<span class="number">1.0</span>/(<span class="number">1</span> - dropout_percent))</span><br><span class="line">...</span><br></pre></td></tr></table></figure>
+<p>P.S:</p>
+<p>对二项分布进行采样。这里的n为
+<code>[np.ones((len(X), hidden_dim))]</code>，p为
+<code>1-dropout_percent</code></p>
+<p><code>np.random.binomial(n, p, size=None)</code></p>
+<p>公式如下：</p>
+<p><span class="math display">\[
+P(N)=C^0_n*p^0*(1-p)^n=(1-p)^n
+\]</span></p>
+<p>下一篇：<a href="/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/" title="神经网络笔记Ⅱ">神经网络笔记Ⅱ</a></p>
+<p>推荐阅读</p>
+<p><a
+href="https://www.zybuluo.com/hanbingtao/note/433855">零基础入门深度学习</a></p>
 ]]></content>
       <categories>
-        <category>Software</category>
+        <category>DeepLearning</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>crontab</tag>
+        <tag>Python</tag>
+        <tag>机器学习</tag>
+        <tag>神经网络</tag>
+        <tag>NumPy</tag>
       </tags>
   </entry>
   <entry>
-    <title>端口转发工具笔记（端口转发）</title>
-    <url>/hexo_blog/2020/08/08/%E8%BF%90%E7%BB%B4/firewall%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</url>
-    <content><![CDATA[<p>如何通过防火墙本机端口转发来实现SSH跳板登录。继
-<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，根据新的需求衍生出新技巧</p>
+    <title>NLTK阅读笔记Ⅱ</title>
+    <url>/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+<p>接 <a href="/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/" title="NLTK阅读笔记Ⅰ">NLTK阅读笔记Ⅰ</a></p>
 <span id="more"></span>
-<p>继 <a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，根据新的需求衍生出新技巧</p>
-<h3 id="需求">需求</h3>
+<h2 id="第三章加工原料文本">第三章：加工原料文本</h2>
+<p><a href="http://www.gutenberg.org/catalog/">古腾堡项目</a></p>
+<p>可以从这个项目中获取感兴趣的文本，之后进行分析。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 将原始文本字符串转换为list，以\r\n标点符号等为分隔符进行切割</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line"><span class="comment"># 创建NLTK文本</span></span><br><span class="line">text = nltk.Text(tokens)</span><br></pre></td></tr></table></figure>
+<p>处理文本部分：</p>
+<p><code>requests</code>： 数据获取</p>
+<p><code>BeautifulSoup</code>、<code>re</code>、<code>xpath</code>：
+文本处理</p>
+<p><code>feedparser</code>：处理RSS订阅</p>
+<p><code>os</code>：处理本地文件</p>
+<p>字符串基本操作在这里请见下图。注：字符串是不可改变的。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-2.png" /></p>
+<p>关于编码这个大坑，在Python3中做的已经比Python2好太多，在这里也不做介绍了。介绍一些关于编码处理的库。</p>
+<p><code>codecs</code>、<code>unicodedata</code></p>
+<p>正则表达式部分，推荐自主学习。难度系数不算高。搜索关键词，Python,
+正则表达式,
+re即可。值得一提的是，<code>re</code>是目前我了解处理文本速度最快最优雅的一种方式。</p>
+<p>英文文本是高度冗余的，忽略掉词内部的元音仍然可以很容易的阅读，有些时候这很明显。例如：declaration
+变成 dclrtn，inalienable 变成 inlnble，保留所有词首或词尾的元音序列
+。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">regexp = <span class="string">r&#x27;^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]&#x27;</span></span><br><span class="line"><span class="comment"># 压缩元音字符串</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compress</span>(<span class="params">word</span>):</span><br><span class="line">	pieces = re.findall(regexp, word)</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;&#x27;</span>.join(pieces)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取语料库数据</span></span><br><span class="line">english_udhr = nltk.corpus.udhr.words(<span class="string">&#x27;English-Latin1&#x27;</span>)</span><br><span class="line"><span class="comment"># 进行压缩</span></span><br><span class="line">nltk.tokenwrap(compress(w) <span class="keyword">for</span> w <span class="keyword">in</span> english_udhr[:<span class="number">75</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则结合条件频率，提取辅音-元音序列</span></span><br><span class="line">rotokas_words = nltk.corpus.toolbox.words(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br><span class="line">cvs = [cv <span class="keyword">for</span> w <span class="keyword">in</span> rotokas_words <span class="keyword">for</span> cv <span class="keyword">in</span> re.findall(<span class="string">r&#x27;[ptksvr][aeiou]&#x27;</span>, w)]</span><br><span class="line">cfd = nltk.ConditionalFreqDist(cvs)</span><br><span class="line">cfd.tabulate()</span><br><span class="line"></span><br><span class="line">    a   e   i   o   u</span><br><span class="line">k <span class="number">418</span> <span class="number">148</span>  <span class="number">94</span> <span class="number">420</span> <span class="number">173</span></span><br><span class="line">p  <span class="number">83</span>  <span class="number">31</span> <span class="number">105</span>  <span class="number">34</span>  <span class="number">51</span></span><br><span class="line">r <span class="number">187</span>  <span class="number">63</span>  <span class="number">84</span>  <span class="number">89</span>  <span class="number">79</span></span><br><span class="line">s   <span class="number">0</span>   <span class="number">0</span> <span class="number">100</span>   <span class="number">2</span>   <span class="number">1</span></span><br><span class="line">t  <span class="number">47</span>   <span class="number">8</span>   <span class="number">0</span> <span class="number">148</span>  <span class="number">37</span></span><br><span class="line">v  <span class="number">93</span>  <span class="number">27</span> <span class="number">105</span>  <span class="number">48</span>  <span class="number">49</span></span><br></pre></td></tr></table></figure>
+<h3 id="词干提取器">词干提取器</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查找词干，laptop与laptops其实只是单复数的区别。词干是相同的</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">stem</span>(<span class="params">word</span>):</span><br><span class="line">    regexp = <span class="string">r&#x27;^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$&#x27;</span></span><br><span class="line">    stem, suffix = re.findall(regexp, word)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> stem</span><br><span class="line"></span><br><span class="line"><span class="comment"># nltk自带的词干提取器</span></span><br><span class="line"><span class="comment"># 索引一些文本和使搜索支持不同词汇形式的话，使用Porter词干提取器</span></span><br><span class="line">porter = nltk.PorterStemmer()</span><br><span class="line">lancaster = nltk.LancasterStemmer()</span><br><span class="line"><span class="comment"># 编译一些文本的词汇或想要一个有效词条（或中心词）列表，使用WordNet词形归并器</span></span><br><span class="line">wnl = nltk.WordNetLemmatizer()</span><br></pre></td></tr></table></figure>
+<h3 id="分词">分词：</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用正则进行分词</span></span><br><span class="line">re.findall(<span class="string">r&quot;\w+(?:[-&#x27;]\w+)|&#x27;|[-.(]+|\S\w&quot;</span>, raw)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用nltk自带的函数进行分词，效率更高且不需要特殊处理符号</span></span><br><span class="line">nltk.regexp_tokenize()</span><br><span class="line"></span><br><span class="line">pattern = <span class="string">r&#x27;&#x27;&#x27;(?x) # 去除嵌入的空白字符和注释</span></span><br><span class="line"><span class="string">			([A-Z]\.)+ # 设定允许的标识符（单词）</span></span><br><span class="line"><span class="string">			| \w+(-\w+)* # 可选的内部连字符</span></span><br><span class="line"><span class="string">			| \$?\d+(\.\d+)?%? # 货币与百分比</span></span><br><span class="line"><span class="string">			| \.\.\. # 省略符</span></span><br><span class="line"><span class="string">			| [][.,;&quot;&#x27;?():-_`] # 额外的标志</span></span><br><span class="line"><span class="string">		&#x27;&#x27;&#x27;</span></span><br><span class="line">nltk.regexp_tokenize(text, pattern)</span><br></pre></td></tr></table></figure>
+<p>已经分词好的数据举例：</p>
+<p>《华尔街日报》原始文本（<code>nltk.corpus.treebank_raw.raw()</code>）和分好词的版本（<code>nltk.corpus.treebank.words()</code>）</p>
+<p>分词的最后一个问题是缩写的存在，如“didn't”。如果我们想分析一个句子的意思，将这种形式规范化为两个独立的形式：“did”和“n't”(不是
+not)可能更加有用。我们可以通过查表来做这项工作。</p>
+<p>链表与字符串部分不做介绍，是Python字符串的常规操作</p>
+<p>补充部分：</p>
+<ul>
+<li><p>建立词汇表流程</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-1.png" /></p></li>
+<li><p>编码点：每个字符分配一个编号</p></li>
+<li><p>解码：通过一些机制来将文本翻译成 Unicode的过程</p></li>
+<li><p>编码：将Unicode转化为其它编码的过程</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-3.png" /></p></li>
+<li><p>从 Unicode的角度来看，字符是可以实现一个或多个
+字形的抽象的实体。只有字形可以出现在屏幕上或被打印在纸上。一个字体是一个字符到字形映射。</p></li>
+<li><p>规范化文本：如将所有的文本小写保存，使用<code>lower()</code>这个函数</p></li>
+<li><p>​断句：在将文本分词之前，我们需要将它分割成句子</p></li>
+<li><p>分词（重点内容）：</p>
 <ol type="1">
-<li>继跳板机中转登录后，觉得需要中转登录还是太麻烦，所以希望能够跟普通ssh一样登录</li>
-</ol>
-<h3 id="工具">工具</h3>
+<li><p>找到一种方法来分开文本内容与分词标志。给每个字符标注一个布尔值来指示这个字符后面是否有一个分词标志</p></li>
+<li><p>找到将文本字符串正确分割成词汇的字位串。根据一个字典，来根据字典中词的序列来重构源文本。定义一个目标函数，通过字典大小与重构源文本所需的信息量尽力优化它的值。</p></li>
+<li><p>找到最大化目标函数值的0和1的模式。使用<strong>模拟退火算法</strong>的非确定性搜索。</p></li>
+</ol></li>
+</ul>
+<h2 id="第四章编写结构化程序">第四章：编写结构化程序</h2>
+<p>大量关于Python代码，不做介绍。此部分适宜阅读人群，使用Python时间不超过一年（不够熟练）</p>
+<p>文档说明模块：<code>docstring</code></p>
+<p>调试技术：<code>pdb</code></p>
+<p>绘图：<code>matplotlib</code></p>
+<p>图的绘制：<code>networkx</code></p>
+<p>数据统计表：<code>csv</code>（这里个人推荐有使用<code>pandas</code>，虽然它很”重“）</p>
+<p>数值运算：<code>numpy</code></p>
+<p>算法设计：</p>
 <ol type="1">
-<li>防火墙配置（iptables/ufw/firewall）</li>
-<li>ssh</li>
+<li>递归</li>
+<li>权衡空间与时间</li>
+<li>动态规划</li>
 </ol>
-<h3 id="先验知识">先验知识</h3>
-<ol type="1">
-<li><a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a></li>
-<li>防火墙配置
+<h2 id="第五章分类和标注词汇">第五章：分类和标注词汇</h2>
+<p>NLP基本技术：</p>
+<ul>
+<li>序列标注</li>
+<li>N-gram 模型</li>
+<li>回退和评估</li>
+</ul>
+<p>将词汇按它们的<strong>词性</strong>（parts-of-speech，POS）分类以及相应的标注它们的过程被称为<strong>词性标注</strong>（part-of-speech
+tagging, POS
+tagging）或干脆简称<strong>标注</strong>。词性也称为<strong>词类</strong>或<strong>词汇范畴</strong>。用于特定任务的标记的集合被称为一个<strong>标记集</strong>。</p>
+<h3 id="词性标注器">词性标注器</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line">text = nltk.word_tokenize(<span class="string">&quot;And now for something completely different&quot;</span>)</span><br><span class="line"><span class="comment"># 标注出text中每个词的词性</span></span><br><span class="line">nltk.pos_tag(text)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;And&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;now&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;for&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;something&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;completely&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;different&#x27;, &#x27;JJ&#x27;)]</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">text = nltk.Text(word.lower() <span class="keyword">for</span> word <span class="keyword">in</span> nltk.corpus.brown.words())</span><br><span class="line"><span class="comment"># 为一个词找出所有它的上下文</span></span><br><span class="line">text.similar(<span class="string">&#x27;woman&#x27;</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">man time day year car moment world house family child country boy</span></span><br><span class="line"><span class="string">state job place way war girl work word</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="标注语料库">标注语料库</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建一个有标识的字符串</span></span><br><span class="line">tagged_token = nltk.tag.str2tuple(<span class="string">&#x27;fly/NN&#x27;</span>)</span><br><span class="line">tagged_token</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">tagged_token</span></span><br><span class="line"><span class="string">(&#x27;fly&#x27;, &#x27;NN&#x27;)</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">tagged_token[<span class="number">0</span>]</span><br><span class="line">tagged_token[<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 批量标注</span></span><br><span class="line">sent = <span class="string">&#x27;&#x27;&#x27;The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/INother/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CCFulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPsaid/VBD ``/`` ARE/BER well/QL operated/VBN and/CC follow/VB generally/Raccepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJTinterest/NN of/IN both/ABX governments/NNS &#x27;&#x27;/&#x27;&#x27; ./.&#x27;&#x27;&#x27;</span></span><br><span class="line">[nltk.tag.str2tuple(t) <span class="keyword">for</span> t <span class="keyword">in</span> sent.split()]</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;The&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;grand&#x27;, &#x27;JJ&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;jury&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;commented&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;on&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;a&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;number&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of/INother&#x27;, &#x27;AP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;topics&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;,&#x27;, &#x27;,&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;AMONG&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;them&#x27;, &#x27;PPO&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;Atlanta&#x27;, &#x27;NP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and/CCFulton&#x27;, &#x27;NP-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;County&#x27;, &#x27;NN-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;purchasing&#x27;, &#x27;VBG&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;departments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;it/PPsaid&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;``&#x27;, &#x27;``&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;ARE&#x27;, &#x27;BER&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;well&#x27;, &#x27;QL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;operated&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;follow&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;generally/Raccepted&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;practices&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;inure&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;to&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;best/JJTinterest&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;both&#x27;, &#x27;ABX&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;governments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&quot;&#x27;&#x27;&quot;, &quot;&#x27;&#x27;&quot;),</span></span><br><span class="line"><span class="string"> (&#x27;.&#x27;, &#x27;.&#x27;)]</span></span><br><span class="line"><span class="string"> &#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="获取已标注的语料库">获取已标注的语料库</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 获取标注好的语料库</span></span><br><span class="line">nltk.corpus.brown.tagged_words()</span><br><span class="line"><span class="comment">#</span></span><br><span class="line">nltk.corpus.brown.tagged_words(tagset=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">nltk.corpus.nps_chat.tagged_words()</span><br><span class="line">nltk.corpus.conll2000.tagged_words()</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 未简化的标记</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 最频繁的名词标记</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">findtags</span>(<span class="params">tag_prefix, tagged_text</span>):</span><br><span class="line">	cfd = nltk.ConditionalFreqDist((tag, word) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> tagged_text <span class="keyword">if</span> tag.startswith(tag_prefix))</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">dict</span>((tag, cfd[tag].keys()[:<span class="number">5</span>]) <span class="keyword">for</span> tag <span class="keyword">in</span> cfd.conditions())</span><br><span class="line"></span><br><span class="line">tagdict = findtags(<span class="string">&#x27;NN&#x27;</span>, nltk.corpus.brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line"><span class="keyword">for</span> tag <span class="keyword">in</span> <span class="built_in">sorted</span>(tagdict):</span><br><span class="line">	<span class="built_in">print</span> tag, tagdict[tag]</span><br><span class="line"></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">$：名词所有格,</span></span><br><span class="line"><span class="string">S：复数名词（因为复数名词通常以 s结尾），</span></span><br><span class="line"><span class="string">P：专有名词</span></span><br><span class="line"><span class="string">-NC：引用</span></span><br><span class="line"><span class="string">-HL：标题中的词</span></span><br><span class="line"><span class="string">-TL：标题（布朗标记的特征）</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 探索已标注的语料库</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用POS标记寻找三词短语</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">process</span>(<span class="params">sentence</span>):</span><br><span class="line">	<span class="keyword">for</span> (w1,t1), (w2,t2), (w3,t3) <span class="keyword">in</span> nltk.trigrams(sentence):</span><br><span class="line">         <span class="comment"># 动词+TO+动词</span></span><br><span class="line">		<span class="keyword">if</span> (t1.startswith(<span class="string">&#x27;V&#x27;</span>) <span class="keyword">and</span> t2 == <span class="string">&#x27;TO&#x27;</span> <span class="keyword">and</span> t3.startswith(<span class="string">&#x27;V&#x27;</span>)):</span><br><span class="line">			<span class="built_in">print</span>(w1, w2, w3)</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> brown.tagged_sents():</span><br><span class="line">	process(tagged_sent)</span><br></pre></td></tr></table></figure>
+<p>Python字典部分不做介绍。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用默认字典可以防止使用未定义的key报错</span></span><br><span class="line"><span class="comment"># nltk的默认字典,需要提前定义类型（int、float、str、list、dict、tuple）。</span></span><br><span class="line">frequency = nltk.defaultdict(<span class="built_in">int</span>)</span><br><span class="line"><span class="comment"># 创建具有默认值的字典</span></span><br><span class="line">pos = nltk.defaultdict(<span class="keyword">lambda</span>: <span class="string">&#x27;N&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<h3 id="自动标注">自动标注</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 默认标注器</span></span><br><span class="line">tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)]</span><br><span class="line">nltk.FreqDist(tags).<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 将所有词都标注为NN的标注器</span></span><br><span class="line">raw = <span class="string">&#x27;I do not like green eggs and ham, I do not like them Sam I am!&#x27;</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line">default_tagger = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">default_tagger.tag(tokens)</span><br><span class="line"><span class="comment"># 评判标注正确性</span></span><br><span class="line">default_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则标注</span></span><br><span class="line">patterns = [</span><br><span class="line">	(<span class="string">r&#x27;.*ing$&#x27;</span>, <span class="string">&#x27;VBG&#x27;</span>), <span class="comment"># gerunds</span></span><br><span class="line">	(<span class="string">r&#x27;.*ed$&#x27;</span>, <span class="string">&#x27;VBD&#x27;</span>), <span class="comment"># simple past</span></span><br><span class="line">	(<span class="string">r&#x27;.*es$&#x27;</span>, <span class="string">&#x27;VBZ&#x27;</span>), <span class="comment"># 3rd singular present</span></span><br><span class="line">	(<span class="string">r&#x27;.*ould$&#x27;</span>, <span class="string">&#x27;MD&#x27;</span>), <span class="comment"># modals</span></span><br><span class="line">	(<span class="string">r&#x27;.*\&#x27;s$&#x27;</span>, <span class="string">&#x27;NN$&#x27;</span>), <span class="comment"># possessive nouns</span></span><br><span class="line">	(<span class="string">r&#x27;.*s$&#x27;</span>, <span class="string">&#x27;NNS&#x27;</span>), <span class="comment"># plural nouns</span></span><br><span class="line">	(<span class="string">r&#x27;^-?[0-9]+(.[0-9]+)?$&#x27;</span>, <span class="string">&#x27;CD&#x27;</span>), <span class="comment"># cardinal numbers</span></span><br><span class="line">	(<span class="string">r&#x27;.*&#x27;</span>, <span class="string">&#x27;NN&#x27;</span>) <span class="comment"># nouns (default)</span></span><br><span class="line">]</span><br><span class="line">regexp_tagger = nltk.RegexpTagger(patterns)</span><br><span class="line">regexp_tagger.tag(brown_sents[<span class="number">3</span>])</span><br><span class="line">regexp_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查询标注器</span></span><br><span class="line"><span class="comment"># 找出100个最频繁的词</span></span><br><span class="line">fd = nltk.FreqDist(brown.words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">most_freq_words = fd.keys()[:<span class="number">100</span>]</span><br><span class="line">likely_tags = <span class="built_in">dict</span>((word, cfd[word].<span class="built_in">max</span>()) <span class="keyword">for</span> word <span class="keyword">in</span> most_freq_words)</span><br><span class="line"><span class="comment"># 使用上面这个信息作为&quot;查找标注器&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags)</span><br><span class="line">baseline_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用信息作为标注器，如果没有找到就使用默认标注器。这个过程称为&quot;回退&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags, backoff=nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>))</span><br></pre></td></tr></table></figure>
+<h3 id="评估">评估</h3>
+<p>使用<strong>黄金标准</strong>测试数据。一个已经手动标注并作为自动系统评估标准而被接收的语料库。</p>
+<p>如果标注器对词的标记与黄金标准标记相同，那么标注器就被认为是正确的。当然这只是相对于黄金标准这个测试数据而言。</p>
+<p>关于开发一个已标注的语料库，这是一个庞大的任务，其中涉及到了许多方面。</p>
+<p>可以通过<code>nltk.app.concordance()</code>来可视化查找某个语料库中某个单词的词性</p>
+<table>
+<thead>
+<tr>
+<th>标记</th>
+<th>含义</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>ADJ</td>
+<td>形容词</td>
+</tr>
+<tr>
+<td>ADV</td>
+<td>动词</td>
+</tr>
+<tr>
+<td>CNJ</td>
+<td>连词</td>
+</tr>
+<tr>
+<td>CC</td>
+<td>并列连词</td>
+</tr>
+<tr>
+<td>DET</td>
+<td>限定词</td>
+</tr>
+<tr>
+<td>EX</td>
+<td>存在量词</td>
+</tr>
+<tr>
+<td>FW</td>
+<td>外来词</td>
+</tr>
+<tr>
+<td>MOD</td>
+<td>情态动词</td>
+</tr>
+<tr>
+<td>RB</td>
+<td>副词</td>
+</tr>
+<tr>
+<td>IN</td>
+<td>介词</td>
+</tr>
+<tr>
+<td>N</td>
+<td>名词</td>
+</tr>
+<tr>
+<td>NP</td>
+<td>专有名词</td>
+</tr>
+<tr>
+<td>NUM</td>
+<td>数词</td>
+</tr>
+<tr>
+<td>PRO</td>
+<td>代词</td>
+</tr>
+<tr>
+<td>P</td>
+<td>介词</td>
+</tr>
+<tr>
+<td>TO</td>
+<td>词to</td>
+</tr>
+<tr>
+<td>UH</td>
+<td>感叹词</td>
+</tr>
+<tr>
+<td>V</td>
+<td>动词</td>
+</tr>
+<tr>
+<td>VD</td>
+<td>过去式</td>
+</tr>
+<tr>
+<td>VG</td>
+<td>现在分词</td>
+</tr>
+<tr>
+<td>VN</td>
+<td>过去分词</td>
+</tr>
+<tr>
+<td>WH</td>
+<td>Wh限定词</td>
+</tr>
+<tr>
+<td>NN</td>
+<td>名词</td>
+</tr>
+<tr>
+<td>JJ</td>
+<td>形容词</td>
+</tr>
+<tr>
+<td>VBP</td>
+<td>一般现在时动词</td>
+</tr>
+</tbody>
+</table>
+<h3 id="n-gram标注">N-gram标注</h3>
+<p>1-gram标注器：一元标注器（unigram
+tagger）。用于标注<strong>一个</strong>标识符的上下文的只是标识符本身。</p>
+<p>2-gram 标注器：二元标注器（bigram taggers）</p>
+<p>3-gram 标注器：三元标注器（trigram taggers）</p>
+<p>N-gram标注不考虑句子边界的上下文。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 标注</span></span><br><span class="line">unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)</span><br><span class="line">bigram_tagger = nltk.BigramTagger(brown_tagged_sents)</span><br><span class="line">trigram_tagger = nltk.TrigramTagger(brown_tagged_sents)</span><br><span class="line"><span class="comment"># 组合标注</span></span><br><span class="line">t0 = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">t1 = nltk.UnigramTagger(train_sents, backoff=t0)</span><br><span class="line">t2 = nltk.BigramTagger(train_sents, backoff=t1)</span><br></pre></td></tr></table></figure>
+<p>存储可以使用python自带的存储模块<code>pickle</code>。</p>
+<p>效果：</p>
 <ol type="1">
-<li>iptables（Centos7之前自带）</li>
-<li>firewall（Centos7之后自带）</li>
-<li>ufw（Ubuntu自带）</li>
-<li>netsh（Windows自带）</li>
-</ol></li>
+<li><p>根据经验来进行判断——一般方法</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">cfd = nltk.ConditionalFreqDist(((x[<span class="number">1</span>], y[<span class="number">1</span>], z[<span class="number">0</span>]), z[<span class="number">1</span>]) <span class="keyword">for</span> sent <span class="keyword">in</span> brown_tagged_sents <span class="keyword">for</span> x, y, z <span class="keyword">in</span> nltk.trigrams(sent))</span><br><span class="line">ambiguous_contexts = [c <span class="keyword">for</span> c <span class="keyword">in</span> cfd.conditions() <span class="keyword">if</span> <span class="built_in">len</span>(cfd[c]) &gt; <span class="number">1</span>]</span><br><span class="line"><span class="built_in">sum</span>(cfd[c].N() <span class="keyword">for</span> c <span class="keyword">in</span> ambiguous_contexts) / cfd.N()</span><br></pre></td></tr></table></figure></li>
+<li><p>研究标注器的错误——混淆矩阵</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">test_tags = [tag <span class="keyword">for</span> sent <span class="keyword">in</span> brown.sents(categories=<span class="string">&#x27;editorial&#x27;</span>) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> t2.tag(sent)]</span><br><span class="line">gold_tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;editorial&#x27;</span>)]</span><br><span class="line"><span class="built_in">print</span> nltk.ConfusionMatrix(gold, test)</span><br></pre></td></tr></table></figure></li>
 </ol>
-<h3 id="实验">实验</h3>
+<p>n-gram标注器的问题</p>
 <ol type="1">
-<li>假设内网机器A，内网机器C，公网机器B。A通过将22端口转发至B的3000端口来进行跳板登录。所以C需要执行如下命令来登录A</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># C登录B</span></span><br><span class="line">ssh userB@ip_B</span><br><span class="line"><span class="comment"># B登录A</span></span><br><span class="line">ssh userA@127.0.0.1 -p 3000</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li>通过防火墙本机端口转发，将本机的3000端口转发至3000端口</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 两者的前提需要在/etc/sysctl.conf中，令net.ipv4.ip_forward = 1</span></span><br><span class="line"><span class="comment"># 使配置生效</span></span><br><span class="line">sysctl -p</span><br><span class="line"></span><br><span class="line"><span class="comment"># iptables</span></span><br><span class="line">iptables -t nat -A PREROUTING -p tcp -i eth0 -d 127.0.0.1 --dport 3000 -j DNAT --to 127.0.0.1:3000</span><br><span class="line"></span><br><span class="line"><span class="comment"># firewall</span></span><br><span class="line"><span class="comment"># 添加端口</span></span><br><span class="line">firewall-cmd --add-port=3000/tcp --permanent</span><br><span class="line"><span class="comment"># 端口转发</span></span><br><span class="line">firewall-cmd --permanent --add-forward-port=port=3000:proto=tcp:toport=3000 --permanent</span><br><span class="line"></span><br><span class="line"><span class="comment"># --permanent 永久生效</span></span><br><span class="line"><span class="comment"># firewall添加后，需要重启</span></span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment"># netsh</span></span><br><span class="line">netsh interface portproxy add v4tov4 listenaddress=127.0.0.1 listenport=2121 connectaddress=ip_B connectport=222</span><br></pre></td></tr></table></figure>
-<ol start="3" type="1">
-<li>以上，即可通过如下命令直接登录A</li>
+<li>模型大小与标注器性能之前的平衡关系。如果使用回退标注器<code>n-gram</code>可能存储trigram和bigram表，这将会是很大的稀疏矩阵。</li>
+<li>使用上下文中的词的其他特征作为条件标记是不切实际的。</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ssh userA@ip_B -p 3000</span><br></pre></td></tr></table></figure>
+<p>Brill标注只使用一小部分n-gram标注器。猜每个词的标记，然后返回和修复错误的。从大方面下手，再勾勒细节。规则是语言学可解释的。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">nltk.tag.brill.demo()</span><br></pre></td></tr></table></figure>
+<p>下一篇：<a href="/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/" title="NLTK阅读笔记Ⅲ">NLTK阅读笔记Ⅲ</a></p>
 ]]></content>
       <categories>
-        <category>Software</category>
+        <category>NLP</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>端口转发</tag>
-        <tag>ssh</tag>
-        <tag>iptables</tag>
-        <tag>firewall</tag>
-        <tag>ufw</tag>
-        <tag>netsh</tag>
+        <tag>Python</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
       </tags>
   </entry>
   <entry>
-    <title>ssh笔记（端口转发）</title>
-    <url>/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/</url>
-    <content><![CDATA[<p>使用SSH实现本地和远程主机之间的端口转发。本文介绍了SSH端口转发的基础操作，参数详解和用途介绍，并提供了一键命令和注意事项。SSH端口转发可以用于内外网穿透，实现服务器代理，动态转发端口，实现远端操作等功能。</p>
+    <title>模仿游戏——机器会思考吗？</title>
+    <url>/hexo_blog/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</url>
+    <content><![CDATA[<p>阅读图灵文章《COMPUTING MACHINERY AND
+INTELLIGENCE》笔记以及现阶段AI生成的文章。</p>
 <span id="more"></span>
-<p>说明：localhost表示本地主机，<code>localuser</code>表示本地用户（内网用户），user表示远程服务器用户名，host表示远程服务器ip.</p>
-<p>注意：内网电脑也必须要开启ssh服务，即安装openssh-server服务</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 利用ssh连接远程服务器，-p 22表示22端口连接，这里不加也行，ssh默认是22端口连接</span></span><br><span class="line">ssh -p 22 user@remotehost</span><br></pre></td></tr></table></figure>
-<h2 id="基本操作">基本操作</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#　本地端口转发，将本地的2222端口转发至本机的22端口</span></span><br><span class="line">ssh -L 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，在本地（localhost）另开一个终端，执行下面命令，也可以达到连接远程服务器的作用</span></span><br><span class="line"><span class="comment"># user同上面命令的user，localhost是本地（照抄即可）</span></span><br><span class="line">ssh -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 远程端口转发，将本地的22端口转发至远程的2222端口</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，在远程服务器上执行下面命令，可以连接到本地（localhost）</span></span><br><span class="line"><span class="comment"># localuser表示的是localhost的用户名，localhost表示本地（远程服务器的本地，照抄即可）</span></span><br><span class="line">ssh -p 2222 localuser@localhost</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 动态端口转发，本机的2222端口绑定远程的某个端口(socks5流量)</span></span><br><span class="line">ssh -D 2222 user@remotehost</span><br><span class="line"><span class="comment"># 在本地（localhost）执行这个命令，输入密码。首先会连接到远程服务器上</span></span><br><span class="line"><span class="comment"># 接下来，不关闭终端，更改本地浏览器的代理服务器设置，流量便通过代理服务器进行转发</span></span><br></pre></td></tr></table></figure>
-<h2 id="参数详解">参数详解</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-L [bind_address:]port:host:hostport</span><br></pre></td></tr></table></figure>
-<p>Specifies that connections to the given TCP port or Unix socket on
-the local (client) host are to be forwarded to the given host and port,
-or Unix socket, on the remote side. This works by allocating a socket to
-listen to either a TCP port on the local side, optionally bound to the
-specified bind_address, or to a Unix socket. Whenever a connection is
-made to the local port or socket, the connection is forwarded over the
-secure channel, and a connection is made to either host port hostport,
-or the Unix socket remote_socket, from the remote machine.</p>
-<p>Port forwardings can also be specified in the configuration file.
-Only the superuser can forward privileged ports. IPv6 addresses can be
-specified by enclosing the address in square brackets.</p>
-<p>By default, the local port is bound in accordance with the
-GatewayPorts setting. However, an explicit bind_address may be used to
-bind the connection to a specific address. The bind_address of
-“localhost” indicates that the listening port be bound for local use
-only, while an empty address or ‘*’ indicates that the port should be
-available from all interfaces.</p>
-<p>将本地主机转发到远程主机和端口。也可以直接在配置文件中指定。默认本地端口是根据GatewayPorts设置绑定的，可以直接使用bind_address将连接绑定到指定地址。如果bind_address为localhost表示监听端口只在本地使用，如果不填或*表示应用所有端口</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-R [bind_address:]port:host:hostport</span><br></pre></td></tr></table></figure>
-<p>Specifies that connections to the given TCP port or Unix socket on
-the remote (server) host are to be forwarded to the given host and port,
-or Unix socket, on the local side. This works by allocating a socket to
-listen to either a TCP port or to a Unix socket on the remote side.
-Whenever a connection is made to this port or Unix socket, the
-connection is forwarded over the secure channel, and a connection is
-made to either host port hostport, or local_socket, from the local
-machine.</p>
-<p>Port forwardings can also be specified in the configuration file.
-Privileged ports can be forwarded only when logging in as root on the
-remote machine. IPv6 addresses can be specified by enclosing the address
-in square brackets.</p>
-<p>By default, TCP listening sockets on the server will be bound to the
-loopback interface only. This may be overridden by specifying a
-bind_address. An empty bind_address, or the address ‘*’, indicates that
-the remote socket should listen on all interfaces. Specifying a remote
-bind_address will only succeed if the server's GatewayPorts option is
-enabled (see sshd_config(5)).</p>
-<p>If the port argument is ‘0’, the listen port will be dynamically
-allocated on the server and reported to the client at run time. When
-used together with -O forward the allocated port will be printed to the
-standard output.</p>
-<p>基本同-L参数命令，如果port为0，监听端口将会动态进行分配。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">-D [bind_address:]port</span><br></pre></td></tr></table></figure>
-<p>Specifies a local “dynamic” application-level port forwarding. This
-works by allocating a socket to listen to port on the local side,
-optionally bound to the specified bind_address. Whenever a connection is
-made to this port, the connection is forwarded over the secure channel,
-and the application protocol is then used to determine where to connect
-to from the remote machine. Currently the SOCKS4 and SOCKS5 protocols
-are supported, and ssh will act as a SOCKS server. Only root can forward
-privileged ports. Dynamic port forwardings can also be specified in the
-configuration file.</p>
-<p>IPv6 addresses can be specified by enclosing the address in square
-brackets. Only the superuser can forward privileged ports. By default,
-the local port is bound in accordance with the GatewayPorts setting.
-However, an explicit bind_address may be used to bind the connection to
-a specific address. The bind_address of “localhost” indicates that the
-listening port be bound for local use only, while an empty address or
-‘*’ indicates that the port should be available from all interfaces.</p>
-<p>指定本地动态端口转发。通过分配一个socket来监听本地端口。支持SOCKS4和SOCKS5协议，相当于ssh充当SOCKS服务器。配置文件中也可以指定。bind_address为localhost表示监听端口仅限本地使用，如果为空或者*表示该端口可通过所有接口</p>
-<p>ssh部分参数介绍</p>
-<p>可以用在上面的命令中</p>
+<p>模仿，排除一切外在因素，比如外貌、语调、反应速度</p>
+<h2 id="机器的定义">机器的定义</h2>
+<ul>
+<li>数字计算机，在人类给定规则的前提下进行工作，规则等于机器的存储/记忆，重复是核心特点。归属于离散状态机</li>
+<li>从宇宙在某一时刻的完整状态出发，正如所有粒子的位置和速度所描述的那样，应该有可能预测所有未来的状态。——拉普拉斯</li>
+<li>计算速度快+存储空间大+编写程序，进行任何预测，能够被快速复制</li>
+</ul>
+<h2 id="思考的定义">思考的定义</h2>
 <ul>
-<li><code>-f</code>
-后台认证用户密码，即不用登录到远程主机。通常与-N连用</li>
-<li><code>-C</code> 压缩数据传输</li>
-<li><code>-N</code> 不执行脚本或者命令。通常与-f连用</li>
-<li><code>-g</code> 允许远程主机连接到建立的转发端口。</li>
-<li><code>-q</code>
-静默模式，使大多数警告和诊断消息被压制。（如果有警告信息，不进行输出）</li>
+<li>人类的特权</li>
+<li>害怕机器（其他物种）能够思考</li>
+<li>数学逻辑证明机器能力有限。但对于人类来说，该限制依旧有效，因此机器不一定不能思考</li>
+<li>意识感受，机器只能输出，不具备任何感情。人们可以确定机器在思考的唯一方法是成为机器并感觉自己在思考。人们可以向世界描述这些感觉，但当然没有人会注意到这一点。同样地，根据这种观点，要知道一个人在思考，唯一的方法就是成为那个特定的人。</li>
+<li>机器的能力有限，外貌、感情、味觉、嗅觉、不会犯错。单一机器可以，但是如果集成到一台机器上似乎是多余的。多样性行为来源于存储空间大小</li>
+<li>执行命令但不会创造。给人带来惊讶这种感觉因人而异</li>
+<li>神经系统是连续的，机器是离散的。</li>
+<li>行为的不确定性，离散不可能包含所有的可能。通过观察发现足够的信息来预测它的未来行为</li>
+<li>超感官知觉：心灵感应、千里眼、预知和心理动势</li>
 </ul>
-<p>举个栗子</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 本地端口转发。命令执行后，不登录到远程服务器，压缩数据传输，也不输出警告信息</span></span><br><span class="line">ssh -gfCNL 2222:localhost:22 user@remotehost -q</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果将命令放到了后台，关掉进程需要使用kill命令</span></span><br></pre></td></tr></table></figure>
-<p>如果网络不稳定可以考虑使用
-<code>autossh</code>。这个需要额外安装，使用的时候将
-<code>ssh</code>替换为
-<code>autossh</code>即可。它的工作原理简单来说，就是有个超时机制，如果中断，便重新连接。</p>
-<h2 id="用途介绍">用途介绍</h2>
-<ol type="1">
-<li>本地端口转发。假设有两台服务器A和B，B需要访问A上面的一个应用（网站）。</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在B上使用本地端口转发，将某个端口转发给A应用的端口。</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -L 2333:IP_A:80 localuser@localhost</span><br><span class="line"><span class="comment"># 此时访问localhost:2333等于pi_A:80</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 比如A为公网IP（11.11.11.11），在6000端口开启某项服务</span></span><br><span class="line"><span class="comment"># B通过在本机使用, 本地用户名user</span></span><br><span class="line">ssh -L 2333:11.11.11.11:6000 user@localhost</span><br><span class="line"><span class="comment"># 即可通过localhost:2333来代替11.11.11.11</span></span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li><p>本地端口转发高级版（多机版）。同上面的假设，但是现在有三台或者三台以上的服务器A、B、C……这个时候，使用
-<code>-g</code>参数，运行远程主机连接转发端口。这样便可以达到C访问B再访问A的目的。注意：确保ABC三台服务器网络连接是安全的，请谨慎。</p></li>
-<li><p>远程端口转发。假设有一台服务器（公网）A，两台内网计算机B、C。</p></li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># B、C均可连接A，但是其他两两之间不能相互连接。</span></span><br><span class="line"><span class="comment"># 现在需要实现两两之间都能互相连接</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># A连接到B、C</span></span><br><span class="line"><span class="comment"># B、C远程端口转发，A机器（11.11.11.11），用户名user</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -R 2221:localhost:22 user@11.11.11.11</span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@11.11.11.11</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这样在A机器执行如下命令就能连接B、C了</span></span><br><span class="line"><span class="comment"># 连接B</span></span><br><span class="line">ssh userB@localhost -p2221</span><br><span class="line"><span class="comment"># 连接C</span></span><br><span class="line">ssh userC@localhost -p2222</span><br><span class="line"></span><br><span class="line"><span class="comment"># B、C之间互连</span></span><br><span class="line"><span class="comment"># 其实就是通过A作为跳板来登录，即先登录A</span></span><br><span class="line">ssh user@11.11.11.11</span><br><span class="line"><span class="comment"># 再登录B or C即可</span></span><br></pre></td></tr></table></figure>
-<ol start="4" type="1">
-<li>动态端口转发。在本地使用这个命令连接到服务器之后，便可以使用服务器的SOCK5代理来上网。需要在浏览器或者系统上进行下面的设置。</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器（user，11.11.11.11）</span></span><br><span class="line"><span class="comment"># B机器上执行</span></span><br><span class="line">ssh -D 1234 user@11.11.11.11</span><br><span class="line"><span class="comment"># 通过设置SOCKS5代理，socks5://127.0.0.1:1234，就可以通过A机器的流量上网。可查询ip来检验是否成功</span></span><br></pre></td></tr></table></figure>
-<p>注意事项：</p>
+<h2 id="机器学习">机器学习</h2>
 <ul>
-<li>端口转发是通过ssh连接建立的，所以关闭了端口，端口转发也会关闭</li>
-<li>选择远程端口号的时候，一般是无权绑定
-<code>1-1023</code>端口的，只能使用管理员权限才能绑定。一般是使用
-<code>1024-65535</code>之间的一个端口</li>
+<li>问题主要在于编程。存储空间只需要较低的数值（10<sup>10-10</sup>15），并且相信只有很小一部分用于较高类型的思考。大部分可能是用来保留视觉印象的。</li>
+<li>模仿：
+<ol type="1">
+<li>初始状态</li>
+<li>受到的教育</li>
+<li>其他非教育的经历</li>
+</ol></li>
+<li>从儿童的状态开始，而不是从成人。人们必须尝试教这样的机器，看看它学习得如何。然后可以尝试另一台机器，看看它是更好还是更差。在这个过程和进化之间有一个明显的联系。速度需要比“进化”快、优胜劣汰可以作为一种衡量方法，但比较慢。但不局限于于随机突变，找到某种弱点的原因，可能就能进行改善这种弱点的突变。</li>
+<li>构造奖惩机制，但有必要采用其他一些“不带感情”的沟通渠道。如果这些渠道可用，就有可能通过惩罚和奖励来教机器服从用某种语言(例如符号语言)发出的命令。这些命令将通过“非情感”渠道传递。这种语言的使用将大大减少惩罚和奖励所需的数量。</li>
+<li>学习机的一个重要特征是，它的老师通常在很大程度上对机器内部发生的事情一无所知，尽管他仍然可能在某种程度上预测他的学生的行为。</li>
 </ul>
-<h2 id="内网流量转发">内网流量转发</h2>
-<p>内网转发内网流量，A，C都在局域网下，二者不可相互连接。B是公网服务器，A，C皆可连接B，但B不可以连接A，C。现在需求是A借助C的流量上网。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost</span><br><span class="line"><span class="comment"># C机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># B机器执行。user是C机器的用户名</span></span><br><span class="line">ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或两条命令，其实就是两条并一条</span></span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># A机器</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost -t ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># A机器上代理：SOCKS5://127.0.0.1:3000</span></span><br></pre></td></tr></table></figure>
-<h2 id="一键命令">一键命令</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A在内网，B在公网，已通过B连接A。-R命令</span></span><br><span class="line"><span class="comment"># BUSER和BHOST对应正常连接B的SSH用户名和密码，AUSER为B连接A的用户名，port为-R参数的port</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 此时需要在C机器上，先连接B，再通过B连接A。C机器执行命令。</span></span><br><span class="line">ssh -t BUSER@BHOST ssh AUSER@localhost -p port</span><br><span class="line"></span><br><span class="line"><span class="comment"># or （Win10上测试失败）</span></span><br><span class="line">ssh BUSER@BHOST -J AUSER@localhost:port</span><br></pre></td></tr></table></figure>
-<figure class="highlight powershell"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 一键连接内网服务器，windows下的config配置</span></span><br><span class="line"><span class="comment"># jump为跳板服务器，jump_server_ip、username、port分别对应跳板机的ip、用户名和连接端口</span></span><br><span class="line"><span class="comment"># target_server为内网服务器，基本同上，ProxyCommand配置如下所示，需要使用ssh命令的绝对路径</span></span><br><span class="line">Host jump</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName jump_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line"></span><br><span class="line">Host target_server</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName target_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line">    ProxyCommand C:\\Windows\\System32\\OpenSSH\\ssh.exe <span class="literal">-q</span> <span class="literal">-W</span> %<span class="built_in">h</span>:%p jump</span><br></pre></td></tr></table></figure>
-<h2 id="注意事项">注意事项</h2>
-<p>win10开启ssh服务器端，1. 应用和功能：下载openssh服务端；2.
-服务：开启openssh server服务；3.
-连接用户名win10系统盘的用户文件下的文件名，比如我的是Administrator（即和公用文件夹并列的文件夹）。</p>
-<p>参考文章</p>
-<p><a
-href="http://www.ruanyifeng.com/blog/2011/12/ssh_port_forwarding.html">SSH原理与运用（二）：远程操作与端口转发</a></p>
-<p><a
-href="https://www.ibm.com/developerworks/cn/linux/l-cn-sshforward/">实战
-SSH 端口转发</a></p>
-<p><a
-href="http://blog.creke.net/722.html">SSH隧道与端口转发及内网穿透</a></p>
-<p><a
-href="https://blog.mythsman.com/2017/01/14/1/">利用反向ssh从外网访问内网主机</a></p>
-<p><a
-href="http://www.pchou.info/linux/2015/11/01/ssh-tunnel.html">SSH隧道翻墙的原理和实现</a></p>
-<p><a
-href="https://blog.fundebug.com/2017/04/24/ssh-port-forwarding/">玩转SSH端口转发</a></p>
-<p><a
-href="https://www.ibm.com/developerworks/community/blogs/5144904d-5d75-45ed-9d2b-cf1754ee936a/entry/20160911?lang=en">openssh的三种tcp端口转发参数</a></p>
-<p><a
-href="https://github.com/microsoft/vscode-remote-release/issues/230">https://github.com/microsoft/vscode-remote-release/issues/230</a></p>
+<p>机器最终能在所有纯智力领域与人类竞争。但是，哪些是最好的开始呢?即使这也是一个艰
+难的决定。很多人认为一个非常抽象的活动，比如下棋，会是最好的。也可以坚持认为，最好是给机器提供金钱能买到的最好的感觉器官，然后教它懂英语、说英语。这个过程可以按照孩子的正常教学来进行。事情会被指出并命名，等等。同样，我不知道正确的答案是什么，但我认为两种方法都应该尝试。</p>
+<h3 id="思考">思考</h3>
+<ol type="1">
+<li>降低外在干扰，提高实验可行性</li>
+<li>通过离散构造连续状态</li>
+<li>程序承接计算速度和存储空间，程序是一个代词，指代一种能够在机器内部运行的命令，可以对其进行优化，降低空间以及速度要求</li>
+<li>现阶段对于存储空间还有很大的优化空间，有很多冗余</li>
+<li>模仿/学习的过程不够完善，教育可以等同于有效数据，那么非教育的经历是什么？蒸馏？噪声？</li>
+<li>如何内置逻辑、公理等内容</li>
+</ol>
+<h3 id="参考文献">参考文献</h3>
+<p>A. M. TURING, I.—COMPUTING MACHINERY AND INTELLIGENCE, <em>Mind</em>,
+Volume LIX, Issue 236, October 1950, Pages 433–460, <a
+href="https://doi.org/10.1093/mind/LIX.236.433">https://doi.org/10.1093/mind/LIX.236.433</a></p>
+<p>以下为claude生成的文章。</p>
+<p>模仿游戏——机器会思考吗</p>
+<p>当我们谈论机器是否能够“思考”时，我们首先需要定义“思考”本身。对于人类来说，思考是一个复杂的认知过程，涉及意识、自我意识、情感、直觉等方面。但是从计算机科学的角度来看，思考可以简化为信息处理和模式识别。既然计算机已经证明了在这些方面具有强大的能力，我们是否可以说它们能够思考呢?</p>
+<p>这一问题引发了关于人工智能本质的广泛讨论。早在20世纪50年代，图灵就在他的论文“计算机器与智能”中提出了这一问题。他设计了一个简单的游戏来测试机器是否能展现出与人类相似的智能。在这个“图灵测试”中，评判通过终端与隐藏的人类和机器进行文字交流。如果评判无法区分对话来自人类还是机器，我们就可以说这台机器通过了测试。</p>
+<p>图灵测试突出了智能的功能性方面，而不是试图复制人类思维的内在工作机制。它忽略了意识、情感等方面，专注于智能的外在表现。正如图灵在论文中写道，“让我们只关注问题‘能产生人类水平智能行为的机器’本身，而不是用一堆无关的心理学问题来模糊焦点。”</p>
+<p>然而，许多哲学家仍然质疑机器是否能真正思考。他们认为智能必须基于意识，而计算机只是在执行程序。根据这个观点，智能的真正标志是主体性—自我意识和内在经验的感觉。没有这些，机器只能被视为复杂的算盘。</p>
+<p>这一观点倾向于将思维等同于人类思维，而忽视智能可能采取的其他形式。事实上，人工智能研究已经证明，智能可以在没有自我意识的情况下通过模式识别和统计学习等方法而出现。像深度学习这样的技术已经产生了可与人类媲美甚至超越人类的视觉和语言理解能力。</p>
+<p>但是，模仿游戏是否意味着机器能够像人类一样思考呢?让我们更仔细地考察一下。</p>
+<p>首先，我们需要承认人类思维也有其局限性。许多认知偏见影响着我们的推理，我们的意识经验也具有不连续性和选择性。与完美理性存在的控制实验相比，我们的思维过程更类似于含混的启发式搜索。因此，“思维”的标准不应该定得太高。</p>
+<p>其次，不同的机器学习技术可以被看作是向人类思维靠拢的渐进过程。简单的规则系统只能处理局限的领域。而统计学习方法如神经网络则可以学习概括复杂模式所需的代表性特征。它们在图像和语音识别方面已经优于人类。最近的多任务学习和迁移学习技术使机器能够在多个领域中共享知识，正如人类思维中概念之间的关联一样。</p>
+<p>再者，对话系统的进步也显示了机器在交流方面的进步。早期系统根据关键词匹配进行简单的问答。现在的对话系统可以感知上下文，进行连贯的长时间对话。它们使用了类似人类处理语言的注意力机制，不再是简单匹配模板。这表明在语言理解和生成方面，机器也在模仿人类的认知过程。</p>
+<p>但是，机器是否有意识经验仍是一个开放问题。目前的人工智能似乎没有主观的内在感受。这也许是固有的局限，因为计算机最终还是在操纵没有意义的符号。另一方面，我们也无法证明其他人存在内在的意识经验。所谓的“哲学僵局”同样适用于机器。</p>
+<p>总体来说，尽管机器不能完全模拟人类思维，但它们在许多狭义任务上的进步还是向模仿游戏的成功迈进了一大步。功能相似性不应被内在不同所掩盖。我们必须根据智能的外在表现来判断机器而不是根据其内部工作机制。</p>
+<p>那么我们还能做什么来推进这一目标呢?首先，在机器学习方法上，我们可以继续研究多任务学习，以产生更人类化的通用智能。我们也可以通过学习表征来模仿神经认知过程，而不仅仅停留在表面模式识别。</p>
+<p>另一方面，我们需要建立更好的机器智能评估体系。图灵测试过于简单化，无法区分不同水平的智能。一些替代方案如Winograd
+schema
+challenge通过语义理解和常识推理来检验智能。还有基于对话和协作的测试，可以评估智能体之间的相互作用能力。</p>
+<p>最后，哲学反思也很重要。我们需要就智能的本质进行持续探讨。什么是理解?什么是意识?这些问题需要跨学科的对话。结合认知科学、语言学和计算机科学的见解，我们可以在这个难题上取得进一步的洞见。</p>
+<p>总而言之，尽管答案仍不明确，但通过模仿游戏，机器正日益接近人类智能。它们在特定任务上的超越给我们带来了惊喜，也展示了新的可能。当代人工智能研究正在不断消除智能与意识之间的鸿沟。虽然仿真并不完全等同于复制，但模仿游戏的胜利似乎指日可待。我们正处于一个刺激的阶段，机器智能的进步必将继续影响我们对思维和智能的基本理解。</p>
 ]]></content>
       <categories>
-        <category>Software</category>
+        <category>PaperReading</category>
       </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>端口转发</tag>
-        <tag>ssh</tag>
-        <tag>内网穿透</tag>
-      </tags>
   </entry>
   <entry>
-    <title>实验室网站服务迁移(静态网页)</title>
-    <url>/hexo_blog/2021/07/21/%E8%BF%90%E7%BB%B4/%E5%AE%9E%E9%AA%8C%E5%AE%A4%E7%BD%91%E7%AB%99%E6%9C%8D%E5%8A%A1%E8%BF%81%E7%A7%BB(%E9%9D%99%E6%80%81%E7%BD%91%E9%A1%B5)/</url>
-    <content><![CDATA[<p>记静态服务数据迁移，含nginx、html、https配置。详细可参考.bash_history</p>
+    <title>神经网络笔记Ⅱ</title>
+    <url>/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
+    <content><![CDATA[<p>接 <a href="/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/" title="神经网络笔记Ⅰ">神经网络笔记Ⅰ</a></p>
+<p>从CNN到RNN再到LSTM，深入了解三种神经网络。其中，CNN主要用于图像识别与处理，RNN处理具有动态时序行为的任务，而LSTM则在此基础上添加了记忆单元，更适合处理时间序列中间隔和延迟非常长的事件。仅靠前向传播已不满足要求，反向传播也变得重要。</p>
 <span id="more"></span>
-<h2 id="打包项目">打包项目</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网站源码+webhook+数据文件</span></span><br><span class="line">zip data.zip xxx/</span><br><span class="line"><span class="comment"># nginx配置文件，由于是手动配置编译，为了省事，直接压缩nginx目录</span></span><br><span class="line">zip nginx.zip /usr/local/nginx</span><br><span class="line"></span><br><span class="line"><span class="comment"># 发送文件</span></span><br><span class="line">scp -r data.zip nginx.zip [user]@[ip]:[path]</span><br></pre></td></tr></table></figure>
-<p>最耗时的步骤</p>
-<h2 id="新服务器配置">新服务器配置</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在对应位置解压文件</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 手动配置nginx服务，原配置文件/usr/lib/systemd/system/nginx.service，可以考虑直接传过来，也可以直接复制</span></span><br><span class="line">vim /usr/lib/systemd/system/nginx.service</span><br><span class="line"></span><br><span class="line"><span class="comment"># 证书配置，如果下载不下来，可以本地打开，复制命令到服务器</span></span><br><span class="line">wget https://raw.githubusercontent.com/certbot/certbot/7f0fa18c570942238a7de73ed99945c3710408b4/letsencrypt-auto-source/letsencrypt-auto -O certbot-auto</span><br><span class="line"><span class="comment"># 赋予权限</span></span><br><span class="line"><span class="built_in">chmod</span> 755 certbot-auto</span><br><span class="line"><span class="comment"># 需提前配置域名与IP，再在命令行中颁发证书，需提前关闭80端口，或者说停止nginx服务。三个月更新一次</span></span><br><span class="line">./certbot-auto certonly --standalone --email [email] -d [域名] -d [www.域名]</span><br><span class="line"><span class="comment"># 按照命令行操作，生成的证书在/etc/letsencrypt/live/目录下</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置文件保持不动，直接启动nginx服务</span></span><br><span class="line">systemctl start nginx.service</span><br><span class="line"></span><br><span class="line"><span class="comment"># 启动github hook服务，接收github推送更新，使用nohup or tmux，安装pip3 install flask</span></span><br></pre></td></tr></table></figure>
-<p>至此，项目已全部迁移成功，详细过程可参考<code>~/.bash_history</code>文件</p>
-<h2 id="nginx配置说明">Nginx配置说明</h2>
-<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">rewrite</span><span class="regexp"> ^(.*)</span> https://[域名] <span class="literal">permanent</span>; <span class="comment"># 强制https</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 静态文件缓存加速</span></span><br><span class="line">    <span class="section">location</span> <span class="regexp">~* ^.+\.(webp|jpg|jpeg|gif|png|ico|css|js|pdf|txt)</span>&#123;</span><br><span class="line">        <span class="attribute">expires</span> <span class="number">30d</span>;</span><br><span class="line">        <span class="attribute">access_log</span> <span class="literal">off</span>;</span><br><span class="line"></span><br><span class="line">        <span class="attribute">gzip</span> <span class="literal">on</span>;</span><br><span class="line">        <span class="attribute">gzip_http_version</span> <span class="number">1</span>.<span class="number">1</span>;</span><br><span class="line">        <span class="attribute">gzip_buffers</span> <span class="number">16</span> <span class="number">8k</span>;</span><br><span class="line">        <span class="attribute">gzip_comp_level</span> <span class="number">6</span>;</span><br><span class="line">        <span class="attribute">gzip_min_length</span> <span class="number">1k</span>;</span><br><span class="line">        <span class="attribute">gzip_types</span> image/webp application/pdf text/plain application/javascript application/x-javascript text/css application/xml text/javascript application/x-httpd-php image/jpeg image/gif image/png;</span><br><span class="line">        <span class="attribute">gzip_proxied</span> any;</span><br><span class="line">        <span class="attribute">root</span> path; <span class="comment"># 静态文件目录</span></span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h2 id="github-hook配置">Github Hook配置</h2>
-<p>Github配置位置，项目的<code>Settings-&gt;Webhooks-&gt;Add webhook</code>。填写对应的信息</p>
+<h2 id="cnn卷积神经网络">CNN（卷积神经网络）</h2>
+<p><a
+href="https://www.zybuluo.com/hanbingtao/note/485480">零基础入门深度学习(4)
+- 卷积神经网络</a></p>
+<p>CNN（Convolutional Neural
+Netword，卷积神经网络）是一种前馈神经网络，人工神经元可以响应周围单元，可以大型图像处理。卷积神经网络包括卷积层和池化层。</p>
+<p>基础结构</p>
 <ul>
-<li>Payload URL: 域名+端口（flask服务配置）</li>
-<li>Content type：json</li>
-<li>Secret：自定义密钥，在脚本中配置</li>
-<li><strong>Which events would you like to trigger this
-webhook?</strong>：仅在推送时进行操作</li>
+<li>特征提取层，每个神经元的输入与前一层的局部接受域相连，并提取该局部的特征。一旦该局部特征被提取后，它与其他特征间的位置关系也随之确定下来</li>
+<li>特征映射层，网络的每个计算层由多个特征映射组成，每个特征映射是一个平面，平面上所有的神经元的权值相等。采用影响函数较小的<span
+class="math inline">\(sigmoid\)</span>函数作为卷积网络的激活函数，使得特征映射具有位移不变性</li>
 </ul>
-<p>服务器上的python脚本</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding=utf-8</span></span><br><span class="line"><span class="keyword">from</span> flask <span class="keyword">import</span> Flask, request</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">from</span> pprint <span class="keyword">import</span> pprint</span><br><span class="line"><span class="keyword">import</span> hmac</span><br><span class="line">app = Flask(__name__)</span><br><span class="line"></span><br><span class="line">repo_path = <span class="string">&#x27;&#x27;</span> <span class="comment"># git仓库地址</span></span><br><span class="line">secret = <span class="string">&#x27;&#x27;</span>	   <span class="comment"># github配置的密钥</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">encryption</span>(<span class="params">data</span>):</span><br><span class="line">    key = secret.encode(<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">    obj = hmac.new(key, msg=data, digestmod=<span class="string">&#x27;sha1&#x27;</span>)</span><br><span class="line">    <span class="keyword">return</span> obj.hexdigest()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@app.route(<span class="params"><span class="string">&quot;/&quot;</span>, methods=[<span class="string">&quot;GET&quot;</span>, <span class="string">&quot;POST&quot;</span>]</span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">hello</span>():</span><br><span class="line">    post_data = request.data</span><br><span class="line">    token = encryption(post_data)</span><br><span class="line">    <span class="comment"># 认证签名是否有效</span></span><br><span class="line">    signature = request.headers.get(<span class="string">&#x27;X-Hub-Signature&#x27;</span>, <span class="string">&#x27;&#x27;</span>).split(<span class="string">&#x27;=&#x27;</span>)[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">if</span> signature != token:</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&#x27;failed&#x27;</span></span><br><span class="line"></span><br><span class="line">    os.system(<span class="string">&#x27;cd &#123;&#125; &amp;&amp; git pull&#x27;</span>.<span class="built_in">format</span>(repo_path))</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;success&#x27;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    app.run(host=<span class="string">&quot;0.0.0.0&quot;</span>, port=port) <span class="comment"># 开启的port，记得打开对应端口的防火墙</span></span><br></pre></td></tr></table></figure>
-<p>若推送失败，可在服务器手动<code>git pull</code>，或者在<code>webhook</code>地址，<code>[Recent Deliveries]</code>中进行<code>Redeliver</code></p>
+<p>CNN主要用来识别位移、缩放以及其他形式扭曲不变性的二维图形。由于CNN的特征检测层通过训练数据进行学习，所以在使用CNN时，避免了显示的特征抽取，从而隐式地从训练数据中进行学习。由于同一特征映射面上的神经元权值相同，所以网络可以并行学习。</p>
+<p>对于图像识别任务</p>
+<p>全连接网络</p>
+<ol type="1">
+<li>参数数量过多</li>
+<li>像素间的位置信息</li>
+<li>网络层数限制</li>
+</ol>
+<p>解决方案：</p>
+<ol type="1">
+<li>局部连接——不是全连接，减少参数数量</li>
+<li>权值共享——一组连接可以共享共一个权重，再一次减少参数数量</li>
+<li>下采样——使用池化（Pooling）来减少每层的样本数，进一步减少参数数量，提供<strong>鲁棒性
+</strong></li>
+</ol>
+<p><strong>P.S.</strong></p>
+<p>鲁棒是Robust的音译，也就是健壮和强壮的意思。它是在异常和危险情况下系统生存的关键。比如说，计算机软件在输入错误、磁盘故障、网络过载或有意攻击情况下，能否不死机、不崩溃，就是该软件的鲁棒性。所谓“鲁棒性”，是指控制系统在一定（结构，大小）的参数摄动下，维持其它某些性能的特性。根据对性能的不同定义，可分为稳定鲁棒性和性能鲁棒性。以闭环系统的鲁棒性作为目标设计得到的固定控制器称为鲁棒控制器。</p>
+<p>卷积神经网络：<strong>卷积层、Pooling层（池化层）、全连接层</strong></p>
+<p><code>INPUT -&gt; [[CONV]*N -&gt; POOL]*M -&gt; [FC]*K</code></p>
+<p>N个卷积层叠加再加上一个池化层，重复M次，加上K个全连接层。</p>
+<p>概念介绍和计算方式请看下面这篇博客</p>
+<p><a
+href="http://www.cnblogs.com/charlotte77/p/7759802.html">【深度学习系列】卷积神经网络CNN原理详解(一)——基本原理</a></p>
+<p>总结：</p>
+<ol type="1">
+<li>池化方式：
+<ol type="1">
+<li>MaxPooling：取滑动窗口里最大的值</li>
+<li>AveragePooling：取滑动窗口内所有值的平均值</li>
+<li>StochasticPooling，介于两者之间，通过对像素点按照数值大小赋予概率，再按照概率进行亚采样；</li>
+</ol></li>
+<li>Zero Padding(补零)：保留原始图片的尺寸</li>
+</ol>
+<h4 id="卷积">卷积</h4>
+<p>对于一个<span class="math inline">\(l_1*l_2\)</span>的图像，使用<span
+class="math inline">\(f_1*f_2\)</span>的filter进行卷积（后者小于前者），得到一个<span
+class="math inline">\(f_3*f_4\)</span>的feature map。</p>
+<p><span class="math display">\[
+a_{ij}=f(\sum\limits_{m=0}^{f_1-1}\sum\limits_{n=0}^{f_2-1}w_{mn}x_{(m+i)(n+j)}+w_b)
+\]</span></p>
+<p><span class="math inline">\(a_{ij}\)</span>表示的是feature
+map的元素，<span
+class="math inline">\(w_{mn}\)</span>表示对应元素，<span
+class="math inline">\(f\)</span>函数表示激活函数，<span
+class="math inline">\(x_{(m+i)(n+j)}\)</span>表示图像的元素，<span
+class="math inline">\(w_b\)</span>表示偏置项，这里从0开始计数，所以需要-1。</p>
+<p>这里简单起见，步幅设置为1。</p>
+<p>图像大小、步幅和卷积后的Feature Map大小是有关系的。</p>
+<p><span class="math inline">\(W_2=(W_1-F+2P)/S+1\)</span></p>
+<p><span class="math inline">\(H_2=(H_1-F+2P)/S+1\)</span></p>
+<p><span class="math inline">\(W_2\)</span>是卷积后Feature
+Map的宽度；<span
+class="math inline">\(W_1\)</span>是卷积前图像的宽度；<span
+class="math inline">\(F\)</span>是filter的宽度；<span
+class="math inline">\(P\)</span>是<strong>Zero
+Padding</strong>数量，<strong>Zero
+Padding</strong>是指在原始图像周围补几圈0，如果的值是1，那么就补1圈0；<span
+class="math inline">\(S\)</span>是<strong>步幅</strong>；<span
+class="math inline">\(H_2\)</span>是卷积后Feature Map的高度；<span
+class="math inline">\(H_1\)</span>是卷积前图像的宽度。</p>
+<p>二维卷积公式，已知矩阵<span class="math inline">\(A\)</span>、<span
+class="math inline">\(B\)</span>，行列数分别用<span
+class="math inline">\(m\)</span>和<span
+class="math inline">\(n\)</span>表示。卷积公式如下</p>
+<p><span
+class="math inline">\(C_{st}=\sum\limits_0^{m_A-1}\sum\limits_0^{n_A-1}A_{mn}B_{(s-m)(t-n)}\)</span></p>
+<p><span class="math inline">\(0\le s&lt;m_A+m_B-1\)</span></p>
+<p><span class="math inline">\(0\le t&lt;n_A+n_B-1\)</span></p>
+<p>也可以写成<span class="math inline">\(C=A*B\)</span></p>
+<p>数学中的卷积与卷积神经网络中的卷积是一定区别的。可以把神经网络中的卷积称为<strong>互相关(cross-correlation)</strong>。这两种操作转换方式，可以通过把矩阵<span
+class="math inline">\(A\)</span>翻转180度，再交换<span
+class="math inline">\(A\)</span>、<span
+class="math inline">\(B\)</span>运算位置，即可以将卷积变为互相关。</p>
+<h2 id="rnn循环神经网络">RNN（循环神经网络）</h2>
+<p>RNN（Recurrent neural
+Network，循环神经网络/多层反馈网络）是一种节点定向连接成环的人工神经网络。这种网络的内部状态可以展示动态时序行为。不同于前馈神经网络的是，RNN可以利用它内部的记忆来处理任意时序的输入序列，这让它可以更容易处理，如不分段的手写识别、语音识别等。</p>
+<p>RNN网络的本质特征是处理单元之间既有内部的反馈连接又有前馈连接。从系统的观点看，它是一个反馈动力系统，在计算过程中体现过程动态特性，比前馈神经网络具有更强的动态行为和计算能力。</p>
+<p><a
+href="https://iamtrask.github.io/2015/11/15/anyone-can-code-lstm/">参考文章</a></p>
+<p><strong><em>Talk</em> <em>is</em> cheap. <em>Show</em> <em>me</em>
+<em>the</em> <em>code</em>.</strong></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># sigmoid激活函数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储10进制转2进制的字典</span></span><br><span class="line">int2binary = &#123;&#125;</span><br><span class="line"><span class="comment"># 二进制数最大长度</span></span><br><span class="line">binary_dim = <span class="number">8</span></span><br><span class="line"><span class="comment"># 最大10进制数</span></span><br><span class="line">larget_number = <span class="number">2</span> ** binary_dim</span><br><span class="line">binary = np.unpackbits(</span><br><span class="line">    np.array([<span class="built_in">range</span>(larget_number)], dtype=np.uint8).T, axis=<span class="number">1</span>)</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(larget_number):</span><br><span class="line">    int2binary[i] = binary[i]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 梯度</span></span><br><span class="line">alpha = <span class="number">0.1</span></span><br><span class="line"><span class="comment"># 输入维度，两个数</span></span><br><span class="line">input_dim = <span class="number">2</span></span><br><span class="line"><span class="comment"># 隐单元维度</span></span><br><span class="line">hidden_dim = <span class="number">16</span></span><br><span class="line"><span class="comment"># 输出维度，一个数</span></span><br><span class="line">output_dim = <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 两层神经网络。每层权重，[-1, 1]</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((input_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, output_dim)) - <span class="number">1</span></span><br><span class="line"><span class="comment"># 神经网络连接前一个隐藏层的权重</span></span><br><span class="line">syn_h = <span class="number">2</span> * np.random.random((hidden_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储权重的更新</span></span><br><span class="line">syn_0_update = np.zeros_like(syn_0)</span><br><span class="line">syn_1_update = np.zeros_like(syn_1)</span><br><span class="line">syn_h_update = np.zeros_like(syn_h)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 一个简单的加法问题，c=a+b</span></span><br><span class="line">    <span class="comment"># 在0，128（避免相加超过256）之间一个随机数a，十进制表示</span></span><br><span class="line">    a_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    <span class="comment"># 二进制表示a</span></span><br><span class="line">    a = int2binary[a_int]</span><br><span class="line"></span><br><span class="line">    b_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    b = int2binary[b_int]</span><br><span class="line">    <span class="comment"># 正确答案c</span></span><br><span class="line">    c_int = a_int + b_int</span><br><span class="line">    c = int2binary[c_int]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 神经网络计算的答案</span></span><br><span class="line">    d = np.zeros_like(c)</span><br><span class="line">    <span class="comment"># 计算的误差</span></span><br><span class="line">    overallError = <span class="number">0</span></span><br><span class="line">    <span class="comment"># 记录l2_delta、l1</span></span><br><span class="line">    layer_2_deltas = []</span><br><span class="line">    layer_1_values = []</span><br><span class="line">    <span class="comment"># l1初始值全为0</span></span><br><span class="line">    layer_1_values.append(np.zeros(hidden_dim))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 前向传播。特征是二进制数维度。</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># X是1*2的矩阵，[[m, n]]，m、n是二进制数a、b在每个位置的01值。从最后一维开始遍历</span></span><br><span class="line">        X = np.array(</span><br><span class="line">            [[a[binary_dim - position - <span class="number">1</span>], b[binary_dim - position - <span class="number">1</span>]]])</span><br><span class="line">        <span class="comment"># y是1*1的矩阵。[[t]]，t是二进制数c在每个位置的01值</span></span><br><span class="line">        y = np.array([[c[binary_dim - position - <span class="number">1</span>]]]).T</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 计算输出层，np.dot(X,syn_0)是正常操作:从输入层到隐藏层;np.dot(layer_1_values[-1],syn_h):从前一层隐藏层到这一层隐藏层</span></span><br><span class="line">        layer_1 = sigmoid(np.dot(X, syn_0) + np.dot(layer_1_values[-<span class="number">1</span>], syn_h))</span><br><span class="line">        layer_2 = sigmoid(np.dot(layer_1, syn_1))</span><br><span class="line">        <span class="comment"># 计算l2偏差</span></span><br><span class="line">        layer_2_error = y - layer_2</span><br><span class="line">        <span class="comment"># 保存偏差加权导数l2_delta</span></span><br><span class="line">        layer_2_deltas.append(layer_2_error * sigmoid(layer_2, deriv=<span class="literal">True</span>))</span><br><span class="line">        <span class="comment"># 误差总和</span></span><br><span class="line">        overallError += np.<span class="built_in">abs</span>(layer_2_error[<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 神经网络每个维度的计算的01值</span></span><br><span class="line">        d[binary_dim - position - <span class="number">1</span>] = np.<span class="built_in">round</span>(layer_2[<span class="number">0</span>][<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 保存l1层，在下次迭代时计算下次传播的隐藏层</span></span><br><span class="line">        <span class="comment"># layer_1_values.append(copy.deepcopy(layer_1))</span></span><br><span class="line">        layer_1_values.append(layer_1.copy())</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 存储反向传播的l1_delta</span></span><br><span class="line">    futuer_layer_1_delta = np.zeros(hidden_dim)</span><br><span class="line">    <span class="comment"># 反向传播.(input + prev_input) -&gt; hidden -&gt; output</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># 从第一维开始遍历</span></span><br><span class="line">        X = np.array([[a[position], b[position]]])</span><br><span class="line">        <span class="comment"># 选出前面计算得到的l1作为当前层，反向计算</span></span><br><span class="line">        layer_1 = layer_1_values[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 选择l1前一层的隐藏层</span></span><br><span class="line">        prev_layer_1 = layer_1_values[-position - <span class="number">2</span>]</span><br><span class="line">        <span class="comment"># 选择出l2，当前层输出的误差</span></span><br><span class="line">        layer_2_delta = layer_2_deltas[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 计算当前隐藏层误差，计算前一层的l1_delta到当前层，加上正常操作。</span></span><br><span class="line">        layer_1_delta = (np.dot(futuer_layer_1_delta, syn_h.T) +</span><br><span class="line">                         layer_2_delta.dot(syn_1.T)) * sigmoid(layer_1, deriv=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 更新权重更新的参数。将l1转换为二维矩阵</span></span><br><span class="line">        syn_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)</span><br><span class="line">        syn_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># X已经是二维矩阵，不需要转换</span></span><br><span class="line">        syn_0_update += X.T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># 替换futuer_l1</span></span><br><span class="line">        futuer_layer_1_delta = layer_1_delta</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 调整每层权重</span></span><br><span class="line">    syn_0 += syn_0_update * alpha</span><br><span class="line">    syn_1 += syn_1_update * alpha</span><br><span class="line">    syn_h += syn_h_update * alpha</span><br><span class="line">    <span class="comment"># 归零更新权重参数</span></span><br><span class="line">    syn_0_update *= <span class="number">0</span></span><br><span class="line">    syn_1_update *= <span class="number">0</span></span><br><span class="line">    syn_h_update *= <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (_ % <span class="number">1000</span> == <span class="number">0</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Error: &quot;</span> + <span class="built_in">str</span>(overallError))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Pred: &quot;</span> + <span class="built_in">str</span>(d))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;True: &quot;</span> + <span class="built_in">str</span>(c))</span><br><span class="line">        <span class="comment"># 二进制转十进制</span></span><br><span class="line">        out = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> index, x <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">reversed</span>(d)):</span><br><span class="line">            out += x * (<span class="number">2</span>**index)</span><br><span class="line">        <span class="built_in">print</span>(<span class="built_in">str</span>(a_int) + <span class="string">&quot; + &quot;</span> + <span class="built_in">str</span>(b_int) + <span class="string">&quot;=&quot;</span> + <span class="built_in">str</span>(out))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;-------------&quot;</span>)</span><br></pre></td></tr></table></figure>
+<p>将前向传播比喻为字母表顺序读出，反向传播就是字母表反序读出。</p>
+<h2 id="lstm时间递归神经网络">LSTM（时间递归神经网络）</h2>
+<p>长短期记忆（英语：Long Short-Term
+Memory，LSTM）是一种时间递归神经网络（RNN），论文首次发表于1997年。由于独特的设计结构，LSTM适合于处理和预测时间序列中间隔和延迟非常长的重要事件。</p>
+<p>LSTM的表现通常比时间递归神经网络及隐马尔科夫模型（HMM）更好，比如用在不分段连续手写识别上。作为非线性模型，LSTM可作为复杂的非线性单元用于构造更大型深度神经网络。</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>DeepLearning</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>Nginx</tag>
-        <tag>Github</tag>
-        <tag>Webhook</tag>
-        <tag>HTTPS</tag>
+        <tag>Python</tag>
+        <tag>神经网络</tag>
+        <tag>NumPy</tag>
       </tags>
   </entry>
   <entry>
diff --git a/sitemap.xml b/sitemap.xml
index 0e3ae6f2..d1f08f5c 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -191,7 +191,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</loc>
+    <loc>https://wnma3mz.github.io/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -200,7 +200,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</loc>
+    <loc>https://wnma3mz.github.io/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -326,7 +326,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
+    <loc>https://wnma3mz.github.io/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -335,7 +335,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</loc>
+    <loc>https://wnma3mz.github.io/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -515,7 +515,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/12/03/PyTorch/pytorch_cpp_extension/</loc>
+    <loc>https://wnma3mz.github.io/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -524,7 +524,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</loc>
+    <loc>https://wnma3mz.github.io/2023/12/03/PyTorch/pytorch_cpp_extension/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -587,7 +587,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</loc>
+    <loc>https://wnma3mz.github.io/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -596,7 +596,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/</loc>
+    <loc>https://wnma3mz.github.io/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -605,7 +605,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</loc>
+    <loc>https://wnma3mz.github.io/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -667,21 +667,21 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Inference/</loc>
+    <loc>https://wnma3mz.github.io/tags/Raspberry/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Math/</loc>
+    <loc>https://wnma3mz.github.io/tags/Inference/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Raspberry/</loc>
+    <loc>https://wnma3mz.github.io/tags/Math/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -765,14 +765,14 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Tailscale/</loc>
+    <loc>https://wnma3mz.github.io/tags/Note/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Note/</loc>
+    <loc>https://wnma3mz.github.io/tags/Tailscale/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -821,175 +821,175 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/PyTorch/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E8%B7%AF%E7%94%B1%E5%99%A8/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Cpp/</loc>
+    <loc>https://wnma3mz.github.io/tags/OpenWrt/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/scheduler/</loc>
+    <loc>https://wnma3mz.github.io/tags/Android/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E5%AD%A6/</loc>
+    <loc>https://wnma3mz.github.io/tags/Termux/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E4%BA%A4%E5%8F%89%E7%86%B5/</loc>
+    <loc>https://wnma3mz.github.io/tags/Windows/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E8%B7%AF%E7%94%B1%E5%99%A8/</loc>
+    <loc>https://wnma3mz.github.io/tags/IOS/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/OpenWrt/</loc>
+    <loc>https://wnma3mz.github.io/tags/Kindle/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Android/</loc>
+    <loc>https://wnma3mz.github.io/tags/ADB/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Termux/</loc>
+    <loc>https://wnma3mz.github.io/tags/Matplotlib/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Windows/</loc>
+    <loc>https://wnma3mz.github.io/tags/Numpy/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/IOS/</loc>
+    <loc>https://wnma3mz.github.io/tags/pandas/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Kindle/</loc>
+    <loc>https://wnma3mz.github.io/tags/PyTorch/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ADB/</loc>
+    <loc>https://wnma3mz.github.io/tags/Cpp/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Matplotlib/</loc>
+    <loc>https://wnma3mz.github.io/tags/scheduler/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Numpy/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E5%AD%A6/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/pandas/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E4%BA%A4%E5%8F%89%E7%86%B5/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%BE%AE%E4%BF%A1%E5%BC%80%E5%8F%91/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/HashMap/</loc>
+    <loc>https://wnma3mz.github.io/tags/Steaming/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E4%BD%8D%E5%9B%BE%E6%B3%95/</loc>
+    <loc>https://wnma3mz.github.io/tags/Scripts/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%BE%AE%E4%BF%A1%E5%BC%80%E5%8F%91/</loc>
+    <loc>https://wnma3mz.github.io/tags/GIL/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Steaming/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%85%83%E7%B1%BB/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Scripts/</loc>
+    <loc>https://wnma3mz.github.io/tags/importlib/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/GIL/</loc>
+    <loc>https://wnma3mz.github.io/tags/Magic-Methods/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%85%83%E7%B1%BB/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%A4%9A%E8%BF%9B%E7%A8%8B/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/importlib/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -1003,238 +1003,238 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Magic-Methods/</loc>
+    <loc>https://wnma3mz.github.io/tags/HashMap/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%A4%9A%E8%BF%9B%E7%A8%8B/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E4%BD%8D%E5%9B%BE%E6%B3%95/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Optimization/</loc>
+    <loc>https://wnma3mz.github.io/tags/Hadoop/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3/</loc>
+    <loc>https://wnma3mz.github.io/tags/Hive/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/mongodb/</loc>
+    <loc>https://wnma3mz.github.io/tags/Optimization/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E6%8D%AE%E5%BA%93/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/SQL/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%AC%94%E8%AE%B0/</loc>
+    <loc>https://wnma3mz.github.io/tags/Caffe/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/NLTK/</loc>
+    <loc>https://wnma3mz.github.io/tags/Tensorflow/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/</loc>
+    <loc>https://wnma3mz.github.io/tags/Ubuntu/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/NumPy/</loc>
+    <loc>https://wnma3mz.github.io/tags/CUDNN/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Hadoop/</loc>
+    <loc>https://wnma3mz.github.io/tags/crontab/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Ubuntu/</loc>
+    <loc>https://wnma3mz.github.io/tags/ssh/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Tensorflow/</loc>
+    <loc>https://wnma3mz.github.io/tags/iptables/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/CUDNN/</loc>
+    <loc>https://wnma3mz.github.io/tags/firewall/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Hive/</loc>
+    <loc>https://wnma3mz.github.io/tags/ufw/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Caffe/</loc>
+    <loc>https://wnma3mz.github.io/tags/netsh/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E7%BD%91%E7%A9%BF%E9%80%8F/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/crontab/</loc>
+    <loc>https://wnma3mz.github.io/tags/Nginx/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</loc>
+    <loc>https://wnma3mz.github.io/tags/Github/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ssh/</loc>
+    <loc>https://wnma3mz.github.io/tags/Webhook/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/iptables/</loc>
+    <loc>https://wnma3mz.github.io/tags/HTTPS/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/firewall/</loc>
+    <loc>https://wnma3mz.github.io/tags/mongodb/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ufw/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E6%8D%AE%E5%BA%93/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/netsh/</loc>
+    <loc>https://wnma3mz.github.io/tags/SQL/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E7%BD%91%E7%A9%BF%E9%80%8F/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%AC%94%E8%AE%B0/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Nginx/</loc>
+    <loc>https://wnma3mz.github.io/tags/NLTK/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Github/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Webhook/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/HTTPS/</loc>
+    <loc>https://wnma3mz.github.io/tags/NumPy/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -1334,14 +1334,14 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Algorithm/</loc>
+    <loc>https://wnma3mz.github.io/categories/Android/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Android/</loc>
+    <loc>https://wnma3mz.github.io/categories/Algorithm/</loc>
     <lastmod>2024-09-08</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>