torch_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0]]],\n",
      "\n",
      "\n",
      "        [[[1]]],\n",
      "\n",
      "\n",
      "        [[[2]]],\n",
      "\n",
      "\n",
      "        [[[3]]],\n",
      "\n",
      "\n",
      "        [[[4]]],\n",
      "\n",
      "\n",
      "        [[[5]]],\n",
      "\n",
      "\n",
      "        [[[6]]],\n",
      "\n",
      "\n",
      "        [[[7]]],\n",
      "\n",
      "\n",
      "        [[[8]]],\n",
      "\n",
      "\n",
      "        [[[9]]]])\n",
      "tensor([[0, 1, 2,  ..., 7, 8, 9],\n",
      "        [0, 1, 2,  ..., 7, 8, 9],\n",
      "        [0, 1, 2,  ..., 7, 8, 9],\n",
      "        ...,\n",
      "        [0, 1, 2,  ..., 7, 8, 9],\n",
      "        [0, 1, 2,  ..., 7, 8, 9],\n",
      "        [0, 1, 2,  ..., 7, 8, 9]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "input_ids = torch.randn((512,10))\n",
    "# print(input_ids)\n",
    "position_ids = torch.arange(10, dtype=torch.long, )\n",
    "print(position_ids.unsqueeze(1).unsqueeze(1).unsqueeze(2))\n",
    "position_ids = position_ids.unsqueeze(0).expand_as(input_ids)\n",
    "print(position_ids)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [
    {
     "data": {
      "text/plain": "nan"
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0)*float('-inf')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[-1.3714, -0.2057,  1.1137,  ..., -0.2960, -0.6635, -0.7693],\n",
      "          [-0.6280,  0.3645, -0.4159,  ...,  0.6154, -0.4032, -0.8866],\n",
      "          [-0.6996,  0.7248, -1.2514,  ...,  2.3442,  1.0420, -0.6090],\n",
      "          ...,\n",
      "          [-0.6644,  1.1190,  1.3489,  ...,  0.0123, -0.3003, -0.5672],\n",
      "          [ 0.4801,  0.7064,  0.4212,  ..., -0.8316, -0.1371,  0.3382],\n",
      "          [-2.2111, -0.1904, -0.1388,  ..., -0.4074, -0.1828,  0.0190]],\n",
      "\n",
      "         [[ 0.5330,  0.6032,  1.8969,  ...,  1.2574,  0.9074,  0.1238],\n",
      "          [-2.9302, -0.2673, -1.7785,  ..., -1.5860, -1.2317, -0.7007],\n",
      "          [-0.1392,  0.0312, -1.3816,  ..., -0.0101, -0.0267, -0.8497],\n",
      "          ...,\n",
      "          [-1.3216,  1.6864,  0.9327,  ..., -1.0344,  0.9940,  1.3416],\n",
      "          [ 2.0405, -0.4577,  0.1687,  ...,  0.7606,  1.2857, -0.9347],\n",
      "          [-1.6194, -0.7169, -1.7255,  ..., -0.8315, -2.6587,  0.2821]],\n",
      "\n",
      "         [[ 2.4463,  0.5014,  0.4900,  ...,  0.4086,  0.1722,  1.3122],\n",
      "          [ 0.6895,  1.2931,  0.7294,  ..., -0.3687, -0.9142, -0.6307],\n",
      "          [ 1.2628,  0.7019, -1.7708,  ...,  0.0949,  0.8177, -0.3380],\n",
      "          ...,\n",
      "          [ 0.9003, -0.0551,  0.1623,  ..., -1.2879, -1.2203, -0.1001],\n",
      "          [ 0.8445,  0.6178,  0.1278,  ..., -0.0574, -0.6314,  1.4794],\n",
      "          [ 1.1730, -1.2129, -1.2609,  ...,  0.4851,  0.7791, -0.4111]],\n",
      "\n",
      "         ...,\n",
      "\n",
      "         [[ 2.1889, -1.4409, -1.1627,  ..., -0.8496,  0.5737, -0.8475],\n",
      "          [-0.6010, -0.0572, -0.6706,  ..., -2.0817,  0.3240,  0.3376],\n",
      "          [-0.1007,  1.1109, -0.0315,  ..., -0.6179, -0.2499,  0.6532],\n",
      "          ...,\n",
      "          [-1.4840, -0.8452,  0.7775,  ..., -0.9389,  0.4858,  0.2004],\n",
      "          [ 0.9397,  0.3318, -0.8291,  ...,  1.1914,  0.2772, -0.9721],\n",
      "          [ 1.9902,  1.5724, -0.8802,  ..., -0.5428,  2.5110, -0.1499]],\n",
      "\n",
      "         [[-0.3657,  2.5728,  0.2070,  ...,  0.7935, -0.3084,  0.9294],\n",
      "          [-0.1406, -0.5131, -2.0563,  ..., -0.4602,  0.5247, -1.3259],\n",
      "          [ 0.6622, -0.0049,  0.9838,  ...,  0.5878, -1.0833,  0.3193],\n",
      "          ...,\n",
      "          [ 0.2950, -1.0737, -0.5763,  ..., -1.5385, -0.2331, -0.6671],\n",
      "          [ 0.7871,  1.2916, -0.7081,  ...,  0.4960,  0.4357, -0.2584],\n",
      "          [-1.2294,  0.1061,  0.1305,  ..., -0.9970, -0.2312, -0.3862]],\n",
      "\n",
      "         [[-0.6942,  1.3621,  0.1383,  ..., -1.3177,  1.4657,  2.3530],\n",
      "          [-2.3476, -0.4397, -1.0540,  ...,  0.1515,  0.4056, -2.1652],\n",
      "          [ 0.8726, -1.0567, -0.7125,  ..., -1.2323,  0.5529,  1.4122],\n",
      "          ...,\n",
      "          [-1.4639,  0.9455, -0.7013,  ..., -0.4259, -1.3648, -1.0692],\n",
      "          [ 1.5017,  1.2216,  1.0048,  ...,  0.4284,  0.3849, -0.5445],\n",
      "          [-1.3125,  2.0719, -0.0179,  ...,  0.1758,  1.3950, -0.4012]]]])\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import torch\n",
    "q = torch.randn((1,23,512,64))\n",
    "k= torch.randn((1,23,256,64))\n",
    "v= torch.randn((1,23,256,64))\n",
    "att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))\n",
    "print(att)\n",
    "# print(input_ids)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2)\n",
      "1\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "def a():\n",
    "    return 1,2\n",
    "a =a()\n",
    "print (a)\n",
    "x,y =a\n",
    "print(x)\n",
    "print(y)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}