Building_RAG_with_MongoDB_and_Amazon_Titan.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWRCEiL2IrnQ"
      },
      "source": [
        "# Step 1: Install libraries\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "akXtl97uIrnR",
        "outputId": "9e55ad88-90be-4546-cc1b-f21458dfa78e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.1/139.1 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.6/12.6 MB\u001b[0m \u001b[31m45.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "! pip install -qU pymongo datasets langchain tiktoken sentence_transformers tqdm boto3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JVSuY5NcIrnR"
      },
      "source": [
        "# Step 2: Setup prerequisites\n",
        "\n",
        "Replace:\n",
        "\n",
        "- `<MONGODB_URI>` with your **MongoDB connection string**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "MG-T5ojXIrnR"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from pymongo import MongoClient"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHqXUOg6IrnS",
        "outputId": "40120ddb-e7cc-455d-ed95-fa3c27a4af5b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'ok': 1}"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Retain the quotes (\"\") when pasting the URI\n",
        "MONGODB_URI = os.environ[\"MONGODB_URI\"]\n",
        "# Initialize a MongoDB Python client\n",
        "mongodb_client = MongoClient(MONGODB_URI, appname=\"devrel.workshop.rag\")\n",
        "# Check the connection to the server\n",
        "mongodb_client.admin.command(\"ping\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5BhCvhS4IrnS"
      },
      "source": [
        "# Step 3: Load the dataset\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "2MAzuLb7IrnT"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from datasets import load_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "eRAVJQ_LIrnT"
      },
      "outputs": [],
      "source": [
        "data = load_dataset(\"si3mshady/aws_whitepapers\", split=\"train\", streaming=True)\n",
        "data_head = data.take(20)\n",
        "docs = pd.DataFrame(data_head).to_dict(\"records\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mnl3fzpgIrnT",
        "outputId": "f9fa0309-7422-4276-9a6a-a551d5d7b0bf"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "20"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check the number of documents in the dataset\n",
        "len(docs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OtdcTiOnIrnT",
        "outputId": "0f8ca8c5-1942-4eb4-b808-ecf5a313ac30"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'Title': 'Amazon_Aurora_Migration_Handbook',\n",
              " 'Content': 'This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 1 Amazon Aurora Migration Handbook July 2020 This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 2 Notices Customers are responsible for making their own independent assessment of the information in this document This document: (a) is for informational purposes only (b) represents current AWS product offerings and practices which are subject to change withou t notice and (c) does not create any commitments or assurances from AWS and its affiliates suppliers or licensors AWS products or services are provided “as is” without warranties representations or conditions of any kind whether express or implied The responsibilities and liabilities of AWS to its customers are controlled by AWS agreements and this document is not part of nor does it modify any agreement between AWS and its customers © 20 20 Amazon Web Services Inc or its affiliates All rights reserved This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 3 Contents Introduction     5 Database Migration Considerations    6 Migration Phases     7 Features and Compatibility    7 Performance     8 Cost      9 Availability and Durability     9 Planning and Testing a Database Migration    11 Homogeneous Migrations     11 Summary of Available Migration Methods    12 Migrating Large Databases to Amazon Aurora   15 Partition and Shard Consolidation on Amazon Aurora   16 MySQL and MySQL compatible Migration Options at a Glance   17 Migrating from Amazon RDS for MySQL    18 Migrating from MySQL Compatible Databases   23 Heterogeneous Migrations    26 Schema Migration     27 Data Migration     28 Example Migration Scenarios    28 SelfManaged Homogeneous Migrations    28 Multi Threaded Migration Using mydumper and myloader   39 Heterogeneous Migrations    45 Testing and Cutover     46 Migration Testing     46 This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 4 Cutover     47 Troubleshooting     49 Troubleshooting MySQL Specific Issues    49 Conclusion     54 Contributors     55 Further Reading     56 Document Revisions     56 This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 5 Abstract This paper outlines the best practices for planning executing and troubleshooting database migrations from MySQL compatible and non  MySQL compatible database products to Amazon Aurora It also teaches Amazon Aurora database administrators how to diagnose and troubleshoot common migration and replication erro rs Introduc tion For decades traditional relational databases have been the primary choice for data storage and persistence These database systems continue to rely on monolithic architectures and were not designed to take advantage of cloud infrastructure These monolithic architectures present many challenges particularly in areas such as cost flexibility and availability In order to address these challenges AWS redesigned relational database for the cloud infrastructure and introduced Amazon Aurora Amazon Aurora is a MySQL compatible relational database engine that combines the speed availability and security of high end commercial databases with the simplicity and cost effectiveness of open source databases  Aurora provides up to five times better performance than MySQL and comparable performance of high end commercial databases Amazon Aurora is priced at one tenth the cost of commercial engines Amazon Aurora is available through the Amazon Relational Database Service (Amazon RDS) platform Like other Amazon RDS databases Aurora is a fully managed database service With the Amazon RDS platform most database management tasks such as hardware provisioning softwa re patching setup configuration monitoring and backup are completely automated Amazon Aurora is built for mission critical workloads and is highly available by default An Aurora database cluster spans multiple Availability Zones (AZs) in a region providing out ofthebox durability and fault tolerance to your data across physical data centers An Availability Zone is composed of one or more highly available data centers operated by Amazon AZs are isolated from each other and are connected through lo w latency links Each segment of your database volume is replicated six times across these AZs This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 6 Aurora cluster volumes automatically grow as the amount of data in your database increases with no performance or availability impact  so there is no need for estimating and provisioning large amount of database storage ahead of time An Aurora cluster volume can grow to a maximum size of 64 terabytes (TB) You are only charged for the space that you use in an Aurora cluster volume Aurora\\'s automated backup capability supports point intime recovery of your data enabling you to restore your database to any second during your retention period up to the last five minutes Automated backups are stored in Amazon Simpl e Storage Service (Amazon S3) which is designed for 99999999999% durability Amazon Aurora backups are automatic incremental and continuous and have no impact on database performance For applications that need read only replicas you can create up to 15 Aurora Replicas per Aurora database with very low replica lag These replicas share the same underlying storage as the source instance lowering costs and avoiding the need to perform writes at the replica nodes Amazon Aurora is highly secure and all ows you to encrypt your databases using keys that you create and control through AWS Key Management Service (AWS KMS) On a database instance running with Amazon Aurora encryption data stored at rest in the underlying storage is encrypted as are the auto mated backups snapshots and replicas in the same cluster Amazon Aurora uses SSL (AES 256) to secure data in transit For a complete list of Aurora features see Amazon Aurora  Given the rich feature se t and cost effectiveness of Amazon Aurora it is increasingly viewed as the go to database for mission critical applications  Database Migration Considerations A database represents a critical component in the architecture of most applications Migrating t he database to a new platform is a significant event in an application’s lifecycle and may have an impact on application functionality performance and reliability You should take a few important considerations into account before embarking on your first migration project to Amazon Aurora Migrations are among the most time consuming and critical tasks handled by database administrators Although the task has become easier with the advent of managed This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 7 migration services such as AWS Database Migration Service  large scale database migrations still require adequate planning and execution to meet strict compatibility and performance requirements Migration Phases Because database migrations tend to be complex we adv ocate taking a phased iterative approach Figure 1  Migration phases This paper examines the following major contributors to the success of every database migration project: • Factors that justify the migration to Amazon Aurora such as compatibility performance cost and high availability and durability • Best practices for choosing the optimal migration method • Best practices for planning and executing a migration • Migration troubleshooting hints This section discusses imp ortant considerations that apply to most database migration projects For an extended discussion of related topics see the Amazon Web Services (AWS) whitepaper Migrating Your Databases to Amazon Aurora  Features and Compatibility Although most applications can be architected to work with many relational database engines you should make sure that your application works with Amazon Aurora Amazon Aurora is designed to be wire compatible with MySQL 5 55657 and 80 Therefore most of the code applications driver s and tools that are used today with MySQL databases can be used with Aurora with little or no change However certain MySQL features like the MyISAM storage engine are not available with Amazon Aurora Also due to the managed nature of the Aurora ser vice SSH This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 8 access to database nodes is restricted which may affect your ability to install third party tools or plugins on the database host For more details see Aurora on Amazon RDS in the Amazon Relational Database Service (Amazon RDS) User Guide  Performance Performance is often the key motivation behind database migrations However deploying your database on Amazon Aurora can be beneficial even if your applications don’t have performance issues For example Amazon Aurora scalability features can greatly reduce the amount of engineering effort that is required to prepare your database platform for future traffic growth You should include benchmarks and performance evaluations in every migration project Therefore many successful database migration projects start with performance evaluations of the new database platform Although the RDS Aurora Performance Assessment Benchmarking paper gives you a decent idea of overall database performance these benchmarks do not emulate the data access patterns of your applications For more useful results test the database performance for time sensitive workloads by running your queries (or subset of your queries) on the new platform directly Consider these strategies : • If your current database is MySQL migrate to Amazon Aurora with downtime and performance test your database with a test or staging version of your application or by replaying the production workload • If you are on a non MySQL compliant engine you can selectively copy the busiest tables to Amazon Aurora and test your queries for t hose tables This gives you a good starting point Of course testing after complete data migration will provide a full picture of real world performance of your application on the new platform Amazon Aurora delivers comparable performance with commercia l engines and significant improvement over MySQL performance It does this by tightly integrating the database engine with an SSD based virtualized storage layer designed for database workloads This reduces writes to the storage system minimizes lock con tention and This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 9 eliminates delays created by database process threads Our tests with SysBench on r38xlarge instances show that Amazon Aurora delivers over 585000 reads per second and 107000 writes per second five times higher than MySQL running the same benchmark on the same hardware One area where Amazon Aurora significantly improves upon traditional MySQL is highly concurrent workloads In order to maximize your workload’s throughput on Amazon Aurora we recommend architecting your applications to driv e a large number of concurrent queries  Cost Amazon Aurora provides consistent high performance together with the security availability and reliability of a commercial database at one tenth the cost Owning and running databases come with associated cost s Before planning a database migration an analysis of the total cost of ownership (TCO ) of the new database platform is imperative Migration to a new database platform should ideally lower the total cost of ownership while providing your applications with similar or better features If you are running an open source database engine (MySQL Postgres) your costs are largely related to hardware server management and database management activities However if you are running a commercial database engine (Oracle SQL Server DB2 etc) a significant portion of your cost is database licensing Amazon Aurora can even be more cost efficient than open source databases because its high scalability helps you reduce the number of database instances that are required to handle the same workload For more details see the Amazon RDS for Aurora Pricing page Availability and Durability High availability and disaster recovery are important considerations for databases Your application may already have very strict recovery time objective (RTO) and recovery point objective (RPO) requirements Amazon Aurora can help you meet or exceed your availability goals by having the following components: This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 10 1 Read replicas : Increase read throughput to support high volume application requests by creating up to 15 database Aurora replicas Amazon Aurora Replicas share the same underlying storage as the source inst ance lowering costs and avoiding the need to perform writes at the replica nodes This frees up more processing power to serve read requests and reduces the replica lag time  often down to single digit milliseconds Aurora provides a reader endpoint so th e application can connect without having to keep track of replicas as they are added and removed Aurora also supports auto scaling where it automatically adds and removes replicas in response to changes in performance metrics that you specify Aurora sup ports cross region read replicas Cross region replicas provide fast local reads to your users and each region can have an additional 15 Aurora replicas to further scale local reads 2 Global Database : You can choose between Global Database  which provides the best replication performance and traditional binlog based replication You can also set up your own binlog replication with external MySQL databases Amazon Aurora Global Database is de signed for globally distributed applications allowing a single Amazon Aurora database to span multiple AWS regions It replicates your data with no impact on database performance enables fast local reads with low latency in each region and provides disa ster recovery from region wide outages 3 Multi AZ: Aurora stores copies of the data in a DB cluster across multiple Availability Zones in a single AWS Region regardless of whether the instances in the DB cluster span multiple Availability Zones For more i nformation on Aurora see Managing an Amazon Aurora DB Cluster  When data is written to the primary DB instance Aurora synchronously replicates the data across Availability Zones to six storage nodes associated with your cluster volume Doing so provides data redundancy eliminates I/O freezes and minimizes latency spikes during system backups Running a DB instance with high availability can enhance availability during planned system maintenance and help protect your databases against failure and Availability Zone disruption For more information about durability and availability features in Amazon Aurora see Aurora on Amazon RDS in the Amazon RDS User Guide This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 11 Planning and Testing a Database Migration After you determine that Amazon Aurora is the right fit for your application the next step is to decide on a migration approach and create a database migration plan Here are the suggested high level steps: 1 Review the available migration techniques described in this document and choose one that satisfies your requirements 2 Prepare a migration plan in the form of a step bystep checklist A checklist ensures that all migration steps are executed in the correct order and that the migration process flow can be controlled (eg suspended or resumed) without the risk of important steps be ing missed 3 Prepare a shadow checklist with rollback procedures Ideally you should be able to roll the migration back to a known consistent state from any point in the migration checklist 4 Use the checklist to perform a test migration and take note of the time required to complete each step If any missing steps are identified add them to the checklist If any issues are identified during the test migration address them and rerun the test migration 5 Test all rollback procedures If any rollback proced ure has not been tested successfully assume that it will not work 6 After you complete the test migration and become fully comfortable with the migration plan execute the migration Homogeneous Migrations Amazon Aurora was designed as a drop in replacement for MySQL 56 It offers a wide range of options for homogeneous migrations (eg migrations from MySQL and MySQL compatible databases) This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 12 Summary of Available Migration Methods This section lists common migration sources and the migration metho ds available to them in order of preference Detailed descriptions step bystep instructions and tips for advanced migration scenarios are available in subsequent sections Common method is widely adopted is built aurora read replica asynchronized wit h source master RDS or self managed MySQL databases Figure 1  Common migration sources and migration methods for Amazon Aurora Amazon RDS Snapshot Migration Compatible sources: • Amazon RDS for MySQL 56 • Amazon RDS for MySQL 51 and 55 (after upgrading to RDS for MySQL 56) Feature highlights: • Managed point andclick service available through the AWS Management Console • Best migration speed and ease of use of all migration methods • Can be used with binary log replication for near zero migration downtime For details see Migrating Data from a MySQL DB Instance to an Amazon Aurora DB Cluster in the Amazon RDS User Guide  This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 13 Percona XtraBackup Compatible sources and limitations : • Onpremises or self managed MySQL 56 in EC2 can be migrated zero downtime migration • You can’t restore into an existing RDS instance using this method • The total size is limited to 6 TB • User accounts functions and stored procedures are not imported automatically Feature highlights: • Managed backup ingestion from Percona XtraBackup files stored in an Amazon Simple Storage Servi ce (Amazon S3) bucket • High performance • Can be used with binary log replication for near zero migration downtime For details see Migrating Data from MySQL by using an Amazon S3 bucket in the Amazon RDS User Guide  SelfManaged Export/Import Compatible sources: • MySQL and MySQL compatible databases such as MySQL MariaDB or Percona Server including managed servers such as Amazon RDS for MySQL or MariaDB • NonMySQL compatible databases DMS Migration Compatible sources: • MySQL compatible and non MySQL compatible databases Feature highlights: • Supports heterogeneous and homogenous migrations This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 14 • Managed point andclick data migration service available through the AWS Management Console • Schemas must be migrated separately • Supports CDC replication for near zero migration downtime For details see What Is AWS Database Migration Service? in the AWS DMS User Guide  For a heterogeneous migration where you are migrating from a database engine other than MySQL to a MySQL datab ase AWS DMS is almost always the best migration tool to use But for homogeneous migration where you are migrating from a MySQL database to a MySQL database native tools can be more effective Using Any MySQL Compatible Database as a Source for AWS DMS: Before you begin to work with a MySQL database as a source for AWS DMS make sure that you the following prerequisites These prerequisites apply to either self  managed or Amazon managed sources You must have an account for AWS DMS that has the Replicati on Admin Role  The role needs the following privileges: • Replication Client: This privilege is required for change data capture (CDC) tasks only In other words full loadonly tasks don’t require this privilege • Replication Slave: This privilege is required for change data capture (CDC) tasks only In other words full loadonly tasks don’t require this privilege • Super: This privilege is required only in MySQL versions before 566 DMS highlights for non MySQL compatible sources: • Requires manual schema conversion from source database format into MySQL  compatible format • Data migration can be performed manually using a universal data format such as comma separated values (CSV) This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 15 • Change data capture (CDC) replication might be possible with third  party tool s for near zero migration downtime Migrating Large Databases to Amazon Aurora Migration of large datasets presents unique challenges in every database migration project Many successful large database migration projects use a combination of the following strategies: • Migration with continuous replication: Large databases typically have extended downtime requirements while moving data from source to target To reduce the downtime you can first load baseline data from source to target and then enable replica tion (using MySQL native tools AWS DMS or third party tools) for changes to catch up • Copy static tables first: If your database relies on large static tables with reference data you may migrate these large tables to the target database before migratin g your active dataset You can leverage AWS DMS to copy tables selectively or export and import these tables manually • Multiphase migration: Migration of large database with thousands of tables can be broken down into multiple phases For example you may move a set of tables with no cross joins queries every weekend until the source database is fully migrated to the target database Note that in order to achieve this you need to make changes in your application to connect to two databases simultaneously while your dataset is on two distinct nodes Although this is not a common migration pattern this is an option nonetheless • Database clean up: Many large databases contain data and tables that remain unused In many cases developers and DBAs keep backup copies of tables in the same database or they just simply forget to drop unused tables Whatever the reason a database migration project p rovides an opportunity to clean up the existing database before the migration If some tables are not being used you might either drop them or archive them to another database You might also delete old data from large tables or archive that data to flat files This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 16 Partition and Shard Consolidation on Amazon Aurora If you are running multiple shards or functional partitions of your database to achieve high performance you have an opportunity to consolidate these partitions or shards on a single Aurora databa se A single Amazon Aurora instance can scale up to 64 TB supports thousands of tables and supports a significantly higher number of reads and writes than a standard MySQL database Consolidating these partitions on a single Aurora instance not only redu ces the total cost of ownership and simplify database management but it also significantly improves performance of cross partition queries • Functional partitions : Functional partitioning means dedicating different nodes to different tasks For example i n an e commerce application you might have one database node serving product catalog data and another database node capturing and processing orders As a result these partitions usually have distinct nonoverlapping schemas o Consolidation strateg y: Migrate each functional partition as a distinct schema to your target Aurora instance If your source database is MySQL compliant use native MySQL tools to migrate the schema and then use AWS DMS to migrate the data either one time or continuously using replication If your source database is non MySQL complaint use AWS Schema Conversion Tool to migrate the schemas to Aurora and use AWS DMS for one time load or continuous replication • Data shards : If you have the same schema with distinct sets of data acros s multiple nodes you are leveraging database sharding For example a high traffic blogging service may shard user activity and data across multiple database shards while keeping the same table schema This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 17 o Consolidation strategy : Since all shards share the sa me database schema you only need to create the target schema once If you are using a MySQL compliant database use native tools to migrate the database schema to Aurora If you are using a non MySQL database use AWS Schema Conversion Tool to migrate the database schema to Aurora Once the database schema has been migrated it is best to stop writes to the database shards and use native tools or an AWS DMS one time data load to migrate an individual shard to Aurora If writes to the application cannot be stopped for an extended period you might still use AWS DMS with replication but only after proper planning and testing  MySQL and MySQL compatible Migration Options at a Glance Source Database Type Migration with Downtime Near zero Downtime Migration Amazon RDS MySQL Option 1: RDS snapshot migration Option 2: Manual migration using native tools* Option 3: Schema migration using native tools and data load using AWS DMS Option 1: Migration using native tools + binlog replication Option 2: RDS snapshot migration + binlog replication Option 3: Schema migration using native tools + AWS DMS for data movement MySQL Amazon EC2 or onpremises Option 1: Schema migration with native tools + AWS DMS for data load Option 1: Schema migration using native tools + A WS DMS to move data Oracle/SQL server Option 1: AWS Schema Conversion Tool + AWS DMS (recommended) Option 2: Manual or third  party tool for schema conversion + manual or thirdparty data load in target Option 1: AWS Schema Conversion Tool + AWS DMS (recommended) Option 2: Manual or third  party tool for schema conversion This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 18 Migrating from Amazon RDS for MySQL If you are migrating from an RDS MySQL 56 database (DB) instance the recommended approach is to use the snapshot migration feature Snapshot m igration is a fully managed point andclick feature that is available through the AWS Management Console You can use it to migrate an RDS MySQL 56 DB instance snapshot into a new Aurora DB cluster It is the fastest and easiest to use of all the migrati on methods described in this document For more information about the snapshot migration feature see Migrating Data to an Amazon Aurora DB Cluster in the Amazon RDS User Guide  This section provides ideas for projects that use the snapshot migration feature The liststyle layout in our example instructions can help you prepare your own migration checklist Estimating Space Requirements for Snapshot Migration When you migrate a snapshot of a MySQL DB instance to an Aurora DB cluster Aurora uses an Am azon Elastic Block Store (Amazon EBS) volume to format the data from the snapshot before migrating it There are some cases where additional space is needed to format the data for migration The two features that can potentially cause space issues during m igration are MyISAM tables and using the ROW_FORMAT=COMPRESSED option If you are not using either of these features in your source database then you can skip this section because you should not have space issues During migration MyISAM tables are conve rted to InnoDB and any compressed tables are uncompressed Consequently there must be adequate room for the additional copies of any such tables  The size of the migration volume is based on the allocated size of the source MySQL database that the snapsho t was made from Therefore if you have MyISAM or compressed tables that make up a small percentage of the overall database size and there is available space in the original database then migration should succeed without encountering any space issues How ever if the original database would not have enough room to store a copy of converted MyISAM tables as well as another (uncompressed) copy of compressed tables then the migration volume will not be big enough In this situation you would need to modify the source Amazon RDS MySQL This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 19 database to increase the database size allocation to make room for the additional copies of these tables take a new snapshot of the database and then migrate the new snapshot When migrating data into your DB cluster observe the following guidelines and limitations: • Although Amazon Aurora supports up to 64 TB of storage the process of migrating a snapshot into an Aurora DB cluster is limited by the size of the Amazon EBS volume of the snapshot and therefore is limited to a m aximum size of 6 TB Non MyISAM tables in the source database can be up to 6 TB in size However due to additional space requirements during conversion make sure that none of the MyISAM and compressed tables being migrated from your MySQL DB instance exc eed 3 TB in size For more information see Migrating Data from an Amazon RDS MySQL DB Instance to an Amazon Aurora MySQL DB Cluster  You might want to modify your d atabase schema (convert MyISAM tables to InnoDB and remove ROW_FORMAT=COMPRESSED ) prior to migrating it into Amazon Aurora This can be helpful in the following cases: • You want to speed up the migration process • You are unsure of how much space you need t o provision • You have attempted to migrate your data and the migration has failed due to a lack of provisioned space Make sure that you are not making these changes in your production Amazon RDS MySQL database but rather on a database instance that was restored from your production snapshot For more details on doing this see Reducing the Amount of Space Required to Migrate Data into Amazon Aurora in the Amazon RDS User Guide  The naming conventions used in this section are as follows: • Source RDS DB instance refers to the RDS MySQL 56 DB instance that you are migrating from • Target Aurora DB cluster refers to the Aurora DB cluster that you are migrating to This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 20 Migrating with Downtime When migration downtime is acceptable you can use the following high level procedure to migrate an RDS MySQL 56 DB instance to Amazon Aurora: 1 Stop all write activity against the source RDS DB instance Database downtime begins here 2 Take a snapshot of the source RDS DB instance 3 Wait until the snapshot shows as Available in the AWS Management Console 4 Use the AWS Management Console to migrate the snapshot to a new Aurora DB cluster For instructions see Migra ting Data to an Amazon Aurora DB Cluster in the Amazon RDS User Guide  5 Wait until the snapshot migration finishes and the target Aurora DB cluster enters the Available state The time to migrate a snapshot primarily depends on the size of the database You can determine it ahead of the production migration by running a test migration 6 Configure applications to connect to the newly created target Aurora DB cluster instead of the source RDS DB instance 7 Resume write activity against the target Aurora DB cluster Database downtime ends here Migrating with Near Zero Downtime If prolonged migration downtime is not acceptable you can perform a near zero downtime migration through a combination of snapshot migration and binary log replication Perform the high level procedure as follows: 1 On the source RDS DB instance ensure that a utomated backups are enabled 2 Create a Read Replica of the source RDS DB instance 3 After you create the Read Replica manually stop replication and obtain binary log coordinates 4 Take a snapshot of the Read Replica This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 21 5 Use the AWS Management Console to migrat e the Read Replica snapshot to a new Aurora DB cluster 6 Wait until snapshot migration finishes and the target Aurora DB cluster enters the Available state 7 On the target Aurora DB cluster configure binary log replication from the source RDS DB instance using the binary log coordinates that you obtained in step 3 8 Wait for the replication to catch up that is for the replication lag to reach zero 9 Begin cut over by stopping all write activity against the source RDS DB instance Application downt ime begins here 10 Verify that there is no outstanding replication lag and then configure applications to connect to the newly created target Aurora DB cluster instead of the source RDS DB instance 11 Complete cut over by resuming write activity Application downtime ends here 12 Terminate replication between the source RDS DB instance and the target Aurora DB cluster For a detailed description of this procedure see Replication Between Aurora and MySQL or Between Aurora and Another Aurora DB Cluster in the Amazon RDS Us er Guide  If you don’t want to set up replication manually you can also create an Aurora Read Replica from a source RDS MySQL 56 DB instance by using the RDS Management Console The RDS automation does the following: 1 Creates a snapshot of the source RDS DB instance 2 Migrates the snapshot to a new Aurora DB cluster 3 Establishes binary log replication between the source RDS DB instance and the target Aurora DB cluster After replication is established you can complete the cut over steps as described previously This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 22 Migrating from Amazon RDS for MySQL Engine Versions Other than 56 Direct snapshot migration is only supported for RDS MySQL 56 DB instance snapshots You can migrate RDS MySQL DB instances that are running other engine versions by u sing the following procedures RDS for MySQL 51 and 55 Follow these steps to migrate RDS MySQL 51 or 55 DB instances to Amazon Aurora: 1 Upgrade the RDS MySQL 51 or 55 DB instance to MySQL 56 • You can upgrade RDS MySQL 55 DB instances directly to MySQL 56 • You must upgrade RDS MySQL 51 DB instances to MySQL 55 first and then to MySQL 56 2 After you upgrade the instance to MySQL 56 test your applications against the upgraded database and address any compatibility or performance co ncerns 3 After your application passes the compatibility and performance tests against MySQL 56 migrate the RDS MySQL 56 DB instance to Amazon Aurora Depending on your requirements choose the Migrating with Downtime or Migrating with Near Zero Downtime procedures described earlier For more information about upgrading RDS MySQL engine versions see Upgrading the MySQL DB Engine in the Amazon RDS User Guide  RDS for MySQL 57 For migrations from RDS MySQL 57 DB instances the snapshot migration approach is not supported because the database engine version ca n’t be downgraded to MySQL 56 In this case we recommend a manual dump andimport procedure for migrating MySQL compatible databases described later in this whitepaper Such a procedure may be slower than snapshot migration but you can still perform it with near zero downtime using binary log replication This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 23 Migrating from MySQL Compatible Databases Moving to Amazon Aurora is still a relatively simple process if you are migrating from an RDS MariaDB instance an RDS MySQL 57 DB instance or a se lf managed MySQL  compatible database such as MySQL MariaDB or Percona Server running on Amazon Elastic Compute Cloud (Amazon EC2) or on  premises There are many techniques you can use to migrate your MySQL compatible database workload to Amazon Aurora This section describes various migration options to help you choose the most optimal solution for your use case Percona XtraBackup Amazon Aurora supports migration from Percona XtraBackup files that are stored in an Amazon S3 bucket Migrating from binar y backup files can be significantly faster than migrating from logical schema and data dumps using tools like mysqldump  Logical imports work by executing SQL commands to re create the schema and data from your source database which involves considerable processing overhead By comparison you can use a more efficient binary ingestion method to ingest Percona XtraBackup files This migration method is compatible with source servers using MySQL versions and 56 Migrating from Percona XtraBackup files invol ves three steps: 1 Use the innobackupex tool to create a backup of the source database 2 Upload backup files to an Amazon S3 bucket 3 Restore backup files through the AWS Management Console For details and step bystep instructions see Migrating data from MySQL by using an Amazon S3 Bucket  in the Amazon RDS User Guide  SelfManaged Export/Import You can use a variety of export/import tools to migrate your data and schema to Amazon Aurora The tools can be described as “MySQL native” because they are either part of a MySQL project or were designed specifically for MySQL  compatible databases This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 24 Examples of native migration tools include the following: 1 MySQL utilities such as mysqldump  mysqlimport  and mysql command  line client 2 Third party utilities such as mydumper and myloader  For details see this mydumper project page  3 Builtin MySQL commands such as SELECT INTO OUTFILE and LOAD DATA INFILE  Native tools are a great option for power users or database administrators who want to maintain full control over the migration process Self managed migrations involve more steps and are typically slower than RDS snapshot or Percona XtraBackup migrations but they offer the best compatibility and flexibility For an in depth discussion of the best practices for self managed migrations see the AWS whitepaper Best Practices for Migrating MySQ L Databases to Amazon Aurora  You can execute a self managed migration with downtime (without replication) or with nearzero downt ime (with binary log replication) SelfManaged Migration with Downtime The high level procedure for migrating to Amazon Aurora from a MySQL  compatible database is as follows: 1 Stop all write activity against the source database Application downtime begin s here 2 Perform a schema and data dump from the source database 3 Import the dump into the target Aurora DB cluster 4 Configure applications to connect to the newly created target Aurora DB cluster instead of the source database 5 Resume write activity Appli cation downtime ends here For an in depth discussion of performance best practices for self managed migrations see the AWS whitepaper Best Practices for Migrating MySQL Databases to Amazon Aurora  This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 25 SelfManaged Migration with Near Zero Downtime The following is the high level procedure for near zero downtime migration into Amazon Aurora from a MySQL compatible database: 1 On the source database enable binary logging and ensure that binary log files are retained for at least the amount of time that is required t o complete the remaining migration steps 2 Perform a schema and data export from the source database Make sure that the export metadata contains binary log coordinates that are required to establish replication at a later time 3 Import the dump into the tar get Aurora DB cluster 4 On the target Aurora DB cluster configure binary log replication from the source database using the binary log coordinates that you obtained in step 2 5 Wait for the replication to catch up that is for the replication lag to reach zero 6 Stop all write activity against the source database instance Application downtime begins here 7 Double check that there is no outstanding replication lag Then configure applications to connect to the newly created target Aurora DB cluster inst ead of the source database 8 Resume write activity Application downtime ends here 9 Terminate replication between the source database and the target Aurora DB cluster For an in depth discussion of performance best practices of self managed migrations see the AWS whitepaper Best Practices for Mig rating MySQL Databases to Amazon Aurora  AWS Database Migration Service AWS Database Migration Service is a managed database migra tion service that is available through the AWS Management Console It can perform a range of tasks from simple migrations with downtime to near zero downtime migrations using CDC replication This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 26 AWS Database Migration Service may be the preferred option if y our source database can’t be migrated using methods described previously such as the RDS MySQL 56 DB snapshot migration Percona XtraBackup migration or native export/import tools AWS Database Migration Service might also be advantageous if your migrat ion project requires advanced data transformations such as the following : • Remapping schema or table names • Advanced data filtering • Migrating and replicating multiple database servers into a single Aurora DB cluster Compared to the migration methods describe d previously AWS DMS carries certain limitations: • It does not migrate secondary schema objects such as indexes foreign key definitions triggers or stored procedures Such objects must be migrated or created manually prior to data migration • The DMS CDC replication uses plain SQL statements from binlog to apply data changes in the target database Therefore it might be slower and more resource  intensive than the native master/slave binary log replication in MySQL For step bystep instructions on how to migrate your database using AWS DMS see the AWS whitepaper Migrating Your Databases to Amazon Aurora  Heterogeneous Migrations If you a re migrating a non MySQL compatible database to Amazon Aurora several options can help you complete the project quickly and easily A heterogeneous migration project can be split into two phases: 1 Schema migration to review and convert the source schema objects (eg tables procedures and triggers) into a MySQL compatible representation 2 Data migration to populate the newly created schema with data contained in the source database Optionally you can use a CDC replication for near zero downtime migratio n This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 27 Schema Migration You must convert database objects such as tables views functions and stored procedures to a MySQL 56 compatible format before you can use them with Amazon Aurora This section describes two main options for converting schema objects  Whichever migration method you choose always make sure that the converted objects are not only compatible with Aurora but also follow MySQL’s best practices for schema design AWS Schema Conversion Tool The AWS Schema Conversion Tool (AWS SCT) can great ly reduce the engineering effort associated with migrations from Oracle Microsoft SQL Server Sybase DB2 Azure SQL Database Terradata Greenplum Vertica Cassandra and PostgreSQL etc AWS SCT can automatically convert the source database schema and a majority of the custom code including views stored procedures and functions to a format compatible with Amazon Aurora Any code that can’t be automatically converted is clearly marked so that it can be processed manually For more information see the AWS Schema Conversion Tool User Guide For step by step instructions on how to convert a non MySQL compatible schema using the AWS Schema Conversion Tool see t he AWS whitepaper Migrating Your Databases to Amazon Aurora  Manual Schema Migration If your source database is not in the scope of SCT comp atible databases you can either manually rewrite your database object definitions or use available third party tools to migrate schema to a format compatible with Amazon Aurora Many applications use data access layers that abstract schema design from business application code In such cases you can consider redesigning your schema objects specifically for Amazon Aurora and adapting the data access layer to the new schema This might require a greater upfront engineering effort but it allows the new s chema to incorporate all the best practices for performance and scalability This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 28 Data Migration After the database objects are successfully converted and migrated to Amazon Aurora it’s time to migrate the data itself The task of moving data from a non MySQL compatible database to Amazon Aurora is best done using AWS DMS AWS DMS supports initial data migration as well as CDC replication After the migration task starts AWS DMS manages all the complexities of the process including data type transformations compression and parallel data transfer The CDC functionality automatically replicates any changes that are made to the source database during the migration process For more information see the AWS Database Migration Service User Guide  For step bystep instructions on how to migrate data from a non MySQL  compatible database into an Amazon Aurora cluster using AWS DMS see the AWS whitepaper Migrating Your Databases to Amazon Aurora  Example Migration Scenarios There are several approaches for performing both self managed homogeneo us migration and heterogeneous migrations SelfManaged Homogeneous Migrations This section provides examples of migration scenarios from self managed MySQL  compatible databases to Amazon Aurora For an in depth discussion of homogeneous migration best pra ctices see the AWS whitepaper Best Practices for Migrating MySQL Databases to Amazon Aurora  Note: If you are migrating from an Amazon RDS MySQL DB instance you can use the RDS snapshot migration feature instead of doing a self  managed migration See the Migrating from Amazon RDS for MySQL section for more details This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 29 Migrating Using Percona XtraBackup One option for migrating data from MySQL to Amazon Aurora is to use the Percona XtraBackup utility For more information about usin g Percona Xtrabackup utility see Migrating Data from an External MySQL Database  in the Amazon RDS User Guide  Approach This scenario uses the Percona XtraBackup utility to take a binary backup of the source MySQL database The backup files are then uploaded to an Amazon S3 bucket and restored into a new Amazon Aurora DB cluster When to Use You can adopt this approach for small  to large scale migrations when the following conditions are met: • The source database is a MySQL 55 or 56 database  • You have administrative system level access to the source database • You are migrating database servers in a 1 to1 fashion: one source MySQL server becomes one new Aurora DB cluster When to Consider Other Options This approach is not currently supported in the following scenarios • Migrating into existing Aurora DB clusters • Migrating multiple source MySQL servers into a single Aurora DB cluster Examples For a step bystep example see Migrating Data from an External MySQL Database  in the Amazon RDS User Guide  OneStep Migration Using mysqldump Another migration option uses the mysqldump utility to migrate data from MySQL to Amazon Aurora This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 30 Approach This scenario uses the mysqldump utility to export schema and data definitions from the source server and import them into the target Auro ra DB cluster in a single step without creating any intermediate dump files When to Use You can adopt this approach for many small scale migrations when the following conditions are met: • The data set is very small (up to 1 2 GB) • The network connection between source and target databases is fast and stable • Migration performance is not critically important and the cost of re  trying the migration is very low • There is no need to do any intermediate schema or data transformations When to Cons ider Other Options This approach might not be an optimal choice if any of the following conditions are true • You are migrating from an RDS MySQL DB instance or a self managed MySQL 55 or 56 database In that case you might get better results with snapsho t migration or Percona XtraBackup respectively For more • details see the Migrating from Amazon RDS for MySQL and Percona XtraBackup sections • It is impossible to establish a network connection from a single client instance to source and target databases due to network architecture or security considerations • The network connection between source and target databases is unstable or very slow • The data set is larger than 10 GB • Migration performance is critically important • An intermediate dump file is required in order to perform schema or data manipulations before you can import the schema/data This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 31 Notes For the sake of simplicity this scenario assumes the following: 1 Migration commands are executed from a client instance running a Linux operating system 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) that is configured to allow connections from the client instance 3 The target Aurora DB cluster already exists and is configured to allow connections from the client instance If you don’t yet have an Aurora DB cluster review the stepbystep cluster launch instructions in the Amazon RDS User Guide 17 4 Export from the source database is performed using a privileged super  user MySQL ac count For simplicity this scenario assumes that the user holds all permissions available in MySQL 5 Import into Amazon Aurora is performed using the Aurora master user account that is the account whose name and password were specified during the cluster launch process Examples The following command when filled with the source and target server and user information migrates data and all objects in the named schema(s) between the source and t arget servers mysqldump host=<source_server_address> \\\\ user=<source_user> \\\\ password=<source_user_password> \\\\ databases <schema(s)> \\\\ singletransaction \\\\ compress | mysql host=<target_cluster_endpoint> \\\\ user=<target_user> \\\\ password=<target_user_password> This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 32 Descriptions of the options and option v alues for the mysqldump command are as follows: • <source_server_address> : DNS name or IP address of the source server • <source_user> : MySQL user account name on the source server • <source_user_password> : MySQL user account password on the source server • <schema(s)> : One or more schema names • <target_cluster_endpoint> : Cluster DNS endpoint of the target Aurora cluster • <target_user> : Aurora master user name • <target_user_password> : Aurora master user password • single transaction : Enforces a consi stent dump from the source database Can be skipped if the source database is not receiving any write traffic • compress : Enables network data compression See the mysqldump docume ntation for more details Example: mysqldump host=source mysqlexamplecom \\\\ user=mysql_admin_user \\\\ password=mysql_user_password \\\\ databases schema1 \\\\ singletransaction \\\\ compress | mysql host=auroracluster xxxxxamazonawscom \\\\ user=aurora_master_user \\\\ password=aurora_user_password Note: This migration approach requires application downtime while the dump and import are in progress You can avoid application downtime by extending the scenario with MySQL binary log replication See the Self Managed Migration with Near Zero Downtime section for more details This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 33 FlatFile Migration Using Files in CSV Format This scenario demonstrates a schema and data migration using flat file dumps that is dumps that do not encapsulate data in SQL statements Many database administrators prefer to use flat files over SQL format files for the following reasons: • Lack of SQL encap sulation results in smaller dump files and reduces processing overhead during import • Flatfile dumps are easier to process using OS level tools; they are also easier to manage (eg split or combine) • Flatfile formats are compatible with a wide range of database engines both SQL and NoSQL  Approach The scenario uses a hybrid migration approach: • Use the mysqldump utility to create a schema only dump in SQL format The dump describes the structure of schema objects (eg tables views and functions) but does not contain data • Use SELECT INTO OUTFILE SQL commands to create dataonly dumps in CSV format The dumps are created in a one filepertable fashion and contain table data only (no schema definitions) The import phase can be executed in two ways: • Traditional approach: Transfer all dump files to an Amazon EC2 instance located in the same AWS Region and Availability Zone as the target Aurora DB cluster After transferring the dump files you can import them into Amazon Aurora using the mysql command line client and LOAD DATA LOCAL INFILE SQL commands for SQL format schema dumps and the flat file data dumps respectively This is the approach that is demonstrated later in this section This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 34 • Alternative approach: Transfer the SQL format schema dumps t o an Amazon EC2 client instance and import them using the mysql command line client You can transfer the flat file data dumps to an Amazon S3 bucket and then import them into Amazon Aurora using LOAD DATA FROM S3 SQL commands For more information including an example of loading data from Amazon S3 see Migrating Data from MySQL by Using an Amazon S3 Bucket  in the Amazon RDS User Guide  When to Use You can adopt this approach for most migration projects where performance and flexibility are important: • You can dump small data sets and import them one table at a time You can also run multiple SELECT INTO OUTFILE and LOAD DATA INFILE operations in parallel for best performance • Data that is stored in flat file dumps is not encapsulated in database  specific SQL statements Therefore it can be handled and processed easily by the systems participating in the data exchange When to Consider Other Options You might choose not to use this approach if any of the following conditions are true: • You are migrating from an RDS MySQL DB instance or a self managed MySQL 56 database In that case you might get better results with snapshot migration or Percona XtraBackup respectively See the Migrating from Amazon RDS for MySQL and Percona XtraBackup sections for more details • The data set is very small and does not require a high performance migration approach • You want the migration process to be as simple as possible and you don’t require any of the performance and flexibility benefits listed earlier Notes To simplify the demons tration this scenario assumes the following: This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 35 1 Migration commands are executed from client instances running a Linux operating system: o Client instance A is located in the source server’s network o Client instance B is located in the same Amazon VPC Availability Zone and Subnet as the target Aurora DB cluster 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) configured to allow connections from client instance A 3 The target Aurora DB cluster already exist s and is configured to allow connections from client instance B If you don’t have an Aurora DB cluster yet review the stepbystep cluster launch instruct ions in the Amazon RDS User Guide  4 Communication is allowed between both client instances 5 Export from the source database is performed using a privileged super user MySQL account For simplicity this scenario assumes that the user holds all permissions available in MySQL 6 Import into Amazon Aurora is performed using the master user account that is the account whose name and password were specified during the cluster launch process Note that this migration approach requires application downtime while t he dump and import are in progress You can avoid application downtime by extending the scenario with MySQL binary log replication See the Self Managed Migration with Near Zero Downtime sectio n for more details Examples In this scenario you migrate a MySQL schema named myschema  The first step of the migration is to create a schema only dump of all objects mysqldump host=<source_server_address> \\\\ user=<source_user> \\\\ password=<source_user_password> \\\\ databases <schema(s)> \\\\ singletransaction \\\\ nodata > myschema_dumpsql This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 36 Descriptions of the options and option values for the mysqldump command are as follows: • <source_server_address> : DNS name or IP address of th e source server • <source_user> : MySQL user account name on the source server • <source_user_password> : MySQL user account password on the source server • <schema(s)> : One or more schema names • <target_cluster_endpoint> : Cluster DNS endpoint of the target Aur ora cluster • <target_user> : Aurora master user name • <target_user_password> : Aurora master user password • single transaction : Enforces a consistent dump from the source database Can be skipped if the source database is not receiving any write traffic • nodata : Creates a schema only dump without row data For more details see mysqldump in the MySQL 56 Reference Manual Example: admin@clientA:~$ mysqldump host=11223344 user=root \\\\ password=pAssw0rd databases myschema \\\\ singletransaction nodata > myschema_dump_schema_onlysql After you complete the schema only dump you can obtain data dumps for each table After logging in to the source MyS QL server use the SELECT INTO OUTFILE statement to dump each table’s data into a separate CSV file admin@clientA:~$ mysql host=11223344 user=root  password=pAssw0rd mysql> show tables from myschema; + + This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 37 | Tables_in_myschema | + + | t1 | | t2 | | t3 | | t4 | + + 4 rows in set (000 sec) mysql> SELECT * INTO OUTFILE \\'/home/admin/dump/myschema_dump_t1csv\\' FIELDS TERMINATED BY \\'\\' OPTIONALLY ENCLOSED BY \\'\"\\' LINES TERMINATED BY \\' \\\\n\\' FROM myschemat1; Query OK 4194304 rows affected (235 sec) (repeat for all remaining tables) For more information about SELECT INTO statement syntax see SELECT  INTO Syntax in the MySQL 56 Reference Manual  After you complete all dump operations the /home/admin/dump directory contains five files: one schema only dump and four data dumps on e per table admin@clientA:~/dump$ ls sh1 total 685M 40K myschema_dump_schema_onlysql 172M myschema_dump_t1csv 172M myschema_dump_t2csv 172M myschema_dump_t3csv 172M myschema_dump_t4csv Next you compress and transfer the files to client instance B located in the same AWS Region and Availability Zone as the target Aurora DB cluster You can use any file transfer method available to you (eg FTP or Amazon S3) This example uses SCP with SSH private key authentication admin@clientA:~/dump$ gzip mysc hema_dump_*csv admin@clientA:~/dump$ scp i sshkeypem myschema_dump_* \\\\ <clientB_ssh_user>@<clientB_address>:/home/ec2 user/ This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 38 After transferring all the files you can decompress them and import the schema and data Import the schema dump first because a ll relevant tables must exist before any data can be inserted into them admin@clientB:~/dump$ gunzip myschema_dump_*csvgz admin@clientB:~$ mysql host=<cluster_endpoint> user=master \\\\ password=pAssw0rd < myschema_dump_schema_onlysql With the schem a objects created the next step is to connect to the Aurora DB cluster endpoint and import the data files Note the following: • The mysql client invocation includes a localinfile parameter which is required to enable support for LOAD DATA LOCAL INFILE commands • Before importing data from dump files use a SET command to disable foreign key constraint checks for the duration of the database session Disabling foreign key checks not only improves import performance but it also lets you import data files in arbitrary order admin@clientB:~$ mysql localinfile host=<cluster_endpoint> \\\\ user=master password=pAssw0rd mysql> SET foreign_key_checks = 0; Query OK 0 rows affected (000 sec) mysql> LOAD DATA LOCAL INFILE \\'/home/ec2  user/myschema_dump_t1csv\\' > INTO TABLE myschemat1 > FIELDS TERMINATED BY \\'\\' OPTIONALLY ENCLOSED BY \\'\"\\' > LINES TERMINATED BY \\' \\\\n\\'; Query OK 4194304 rows affected (1 min 266 sec) Records: 4194304 Deleted: 0 Skipped: 0 Warnings: 0 (repeat for all rema ining CSV files) This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 39 mysql> SET foreign_key_checks = 1; Query OK 0 rows affected (000 sec) That’s it you have imported the schema and data dumps into the Aurora DB cluster You can find more tips and best practices for self managed migrations in the AWS whitepaper Best Practices for Migrating MySQL Databases to Amazon Aurora  Multi Threaded Migration Using mydumper and myloader Mydumper and myloader are popular open source MySQL export/import tools designed to address performance issues associated with the lega cy mysqldump program They operate on SQL format dumps and offer advanced features such as the following: • Dumping and loading data using multiple parallel threads • Creating dump files in a file pertable fashion • Creating chunked dumps in a multiple filespertable fashion • Dumping data and metadata into separate files for easier parsing and management • Configurable transaction size during import • Ability to schedule dumps in regular intervals For more details see the MySQL Data Dumper project page  Approach The scenario uses the mydumper and myloader tools to perform a multi  threaded schema and data migration without the need to manually invoke any SQL commands or desig n custom migration scripts The migration is performed in two steps: This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 40 1 Use the mydumper tool to create a schema and data dump using multiple parallel threads 2 Use the myloader tool to process the dump files and import them into an Aurora DB cluster also in multi threaded fashion Note that mydumper and myloader might not be readily available in the package repository of your Linux/Unix distribution For your convenience the scenario also shows how to build the tools from source code When to Use You can adopt this approach in most migration projects: • The utilities are easy to use and enable database users to perform multi  threaded dumps and imports without the need to develop custom migration scripts • Both tools are highly flexible and have reasonable co nfiguration defaults You can adjust the default configuration to satisfy the requirements of both small  and large scale migrations When to Consider Other Options You might decide not to use this approach if any of the following conditions are true: • You are migrating from an RDS MySQL DB instance or a self managed MySQL 55 or 56 database In that case you might get better results with snapshot migration or Percona XtraBackup respectively See the Migrating from Amazon RDS for MySQL and Percona XtraBackup sections for more details • You can’t use third party software because of operating system limitations • Your data transformation processes require intermediate dump files in a flat file forma t and not an SQL format Notes To simplify the demonstration this scenario assumes the following: This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 41 1 You execute the migration commands from client instances running a Linux operating system: a Client instance A is located in the source server’s network b Clien t instance B is located in the same Amazon VPC Availability Zone and Subnet as the target Aurora cluster 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) configured to allow connections from client instance A 3 The target Aurora DB cluster already exists and is configured to allow connections from client instance B If you don’t have an Aurora DB cluster yet review the stepbystep cluster launch instructions in the Amazon RDS User Guide  4 Communication is allowed between both client instances 5 You perform the export from the source database using a privileged super user MySQL account For simplicity the example assumes that the user holds all permissions available in MySQL 6 You perform the import into Amazon Aurora using the master user account that is the account whose n ame and password were specified during the cluster launch process 7 The Amazon Linux 2016033 operating system is used to demonstrate the configuration and compilation steps for mydumper and myloader  Note : This migration approach requires application down time while the dump and import are in progress You can avoid application downtime by extending the scenario with MySQL binary log replication See the Self Managed Migration with Near Zero Dow ntime section for more details Examples (Preparing Tools) The first step is to obtain and build the mydumper and myloader tools See the MySQL Data Dumper project page for up todate download links and to ensure that tools are prepared on both client instances This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 42 The utilities depend on several packages that you should install first [ec2user@clientA ~]$ sudo yum install glib2 devel mysql56 \\\\ mysql56devel zlib devel pcre devel openssl devel g++ gcc c++ cmake The next steps involve creating a directory to hold the program sources and then fetching and unpacking the source archive [ec2user@clientA ~]$ mkdir mydumper [ec2 user@clientA ~]$ cd mydumper/ [ec2user@clientA mydumper]$ wget https://launchp adnet/mydumper/09/091/+download/mydumper  091targz 20160629 21:39:03 (153 KB/s)  ‘mydumper 091targz’ saved [44463/44463] [ec2user@clientA mydumper]$ tar zxf mydumper 091targz [ec2user@clientA mydumper]$ cd mydumper 091 Next you b uild the binary executables [ec2user@clientA mydumper 091]$ cmake  (…) [ec2user@clientA mydumper 091]$ make Scanning dependencies of target mydumper [ 25%] Building C object CMakeFiles/mydumperdir/mydumperco [ 50%] Building C object CMakeFiles/mydumperdir/server_detectco [ 75%] Building C object CMakeFiles/mydumperdir/g_unix_signalco Linking C executable mydumper [ 75%] Built target mydumper Scanning dependencies of target myloader [100%] Building C object CMakeFiles/myloaderdi r/myloaderco Linking C executable myloader [100%] Built target myloader This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 43 Optionally you can move the binaries to a location defined in the operating system $PATH so that they can be executed more conveniently [ec2user@clientA mydumper 091]$ sudo mv mydumper /usr/local/bin/mydumper [ec2user@clientA mydumper 091]$ sudo mv myloader /usr/local/bin/myloader As a final step confirm that both utilities are available in the system [ec2user@clientA ~]$ mydumper V mydumper 091 built against MySQL 5631 [ec2user@clientA ~]$ myloader V myloader 091 built against MySQL 5631 Examples (Migration) After completing the preparation steps you can perform the migration The mydumper command uses the following basic syntax mydumper h <source_serve r_address> u <source_user> \\\\ p <source_user_password> B <source_schema> \\\\ t <thread_count> o <output_directory> Descriptions of the parameter values are as follows: • <source_server_address> : DNS name or IP address of the source server • <source_user> : MySQL user account name on the source server • <source_user_password> : MySQL user account password on the source server • <source_schema> : Name of the schema to dump • <thread_count> : Number of parallel threads used to dump the data This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 44 • <output_directory> : Name of the directory where dump files should be placed Note : mydumper is a highly customizable data dumping tool For a complete list of supported parameters and their default values use the builtin help mydumper help The example dump is executed as follows [ec2user@clientA ~]$ mydumper h 11223344 u root \\\\ p pAssw0rd B myschema t 4 o myschema_dump/ The operation results in the following files being created in the dump directory [ec2user@clientA ~]$ ls sh1 myschema_dum p/ total 733M 40K metadata 40K myschema schemacreatesql 40K myschemat1 schemasql 184M myschemat1sql 40K myschemat2 schemasql 184M myschemat2sql 40K myschemat3 schemasql 184M myschemat3sql 40K myschemat4 schemasql 184M myschemat4sql The directory contains a collection of metadata files in addition to schema and data dumps You don’t have to manipulate these files directly It’s enough that the directory structure is understood by the myloader tool Compress the entire directory and transfer it to client instance B [ec2user@clientA ~]$ tar czf myschema_dumptargz myschema_dump [ec2user@clientA ~]$ scp i sshkeypem myschema_dumptargz \\\\ <clientB_ssh_user>@<clientB_address>:/home/ec2 user/ This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 45 When the transfer is complete connect to client instance B and verify that the myloader utility is available [ec2user@clientB ~]$ myloader V myloader 091 built against MySQL 5631 Now you can u npack the dump and import it The syntax used for the myloader command is very similar to what you already used for mydumper  The only difference is the d (source directory) parameter replacing the o (target directory) parameter [ec2user@clientB ~]$ tar zxf myschema_dumptargz [ec2user@clientB ~]$ myloader h <cluster_dns_endpoint> \\\\ u master p pAssw0rd B myschema t 4 d myschema_dump/ Useful Tips • The concurrency level (thread count) does not have to be the same for export and import operations A good rule of thumb is to use one thread per server CPU core (for dumps) and one thread per two CPU cores (for imports) • The schema and data dumps produced by mydumper use an SQL format and are compatible with MySQL 56 Although you will typically use the pair of mydumper and myloader tools together for best results technically you can import the dump files from myloader by using any other MySQL compatible client tool You can find more tips and best practices for self managed migrations in t he AWS whitepaper Best Practices for Migrating MySQL Databases to Amazon Aurora  Heterogeneous Migrations For detailed step bystep instructions on how to migrate schema and data from a non  MySQL compatib le database into an Aurora DB cluster using AWS SCT and AWS DMS see the AWS whitepaper Migrating Your Databases to Amazon Aurora  Prior to running migration we suggest you to review Proof of Concept with Aurora to This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 46 understand the volume of data and representative of your production environment as a blueprint  Testing and Cutover Once the schema and data have been successfully migrated from the source database to Amazon Aurora you are no w ready to perform end toend testing of your migration process The testing approach should be refined after each test migration and the final migration plan should include a test plan that ensures adequate testing of the migrated database Migration T esting Test Category Purpose Basic acceptance tests These pre cutover tests should be automatically executed upon completion of the data migration process Their primary purpose is to verify whether the data migration was successful Following are some common outputs from these tests: • Total number of items processed • Total number of items imported • Total number of items skipped • Total number of warnings • Total number of errors If any of these totals reported by the tests deviate from the expec ted values then it means the migration was not successful and the issues need to be resolved before moving to the next step in the process or the next round of testing This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 47 Test Category Purpose Functional tests These post cutover tests exercise the functionality of the applicat ion(s) using Aurora for data storage They include a combination of automated and manual tests The primary purpose of the functional tests is to identify problems in the application caused by the migration of the data to Aurora Nonfunctional tests Thes e post cutover tests assess the nonfunctional characteristics of the application such as performance under varying levels of load User acceptance tests These post cutover tests should be executed by the end users of the application once the final data migration and cutover is complete The purpose of these tests is for the end users to decide if the application is sufficiently usable to meet its primary function in the organization Cutover Once you have completed the final migration and testing it is time to point your application to the Amazon Aurora database This phase of migration is known as cutover If the planning and testing phase has been executed properly cutover should not lead to unexpected issues Precutover Actions • Choose a cutover window: Identify a block of time when you can accomplish cutover to the new database with minimum disruption to the business Normally you would select a low activity period for the database (typically nights and/or weekends) This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 48 • Make sure changes are caught up: If a near zero downtime migration approach was used to replicate database changes from the source to the target database make sure that all database changes are caught up and your target database is not significantly lagging behind the sour ce database • Prepare scripts to make the application configuration changes: In order to accomplish the cutover you need to modify database connection details in your application configuration files Large and complex applications may require updates to co nnection details in multiple places Make sure you have the necessary scripts ready to update the connection configuration quickly and reliably • Stop the application: Stop the application processes on the source database and put the source database in read only mode so that no further writes can be made to the source database If the source database changes aren’t fully caught up with the target database wait for some time while these changes are fully propagated to the target database • Execute pre cutove r tests: Run automated pre cutover tests to make sure that the data migration was successful Cutover • Execute cutover: If pre cutover checks were completed successfully you can now point your application to Amazon Aurora Execute scripts created in the p re cutover phase to change the application configuration to point to the new Aurora database • Start your application: At this point you may start your application If you have an ability to stop users from accessing the application while the application is running exercise that option until you have executed your post cutover checks Post cutover Checks • Execute post cutover tests: Execute predefined automated or manual test cases to make sure your application works as expected with the new database It ’s a good strategy to start testing read only functionality of the database first before executing tests that write to the database This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 49 Enable user access and closely monitor: If your test cases were executed successfully you may give user access to the app lication to complete the migration process Both application and database should be closely monitored at this time Troubleshooting The following sections provide examples of common issues and error messages to help you troubleshoot heterogenous DMS migrat ions Troubleshooting MyS QL Specific Issues The following issues are specific to using AWS DMS with MySQL databases Topics • CDC Task Failing for Amazon RDS DB Instance Endpoint Because Binary Logging Disabled • Connections to a target MySQL instance are disconnected during a task • Adding Autocommit to a MySQL compatible Endpoint • Disable Foreign Keys on a Target MySQL compatible Endpoint • Characters Replaced with Question Mark • \"Bad event\" Log Entries • Change Data Capture with MySQL 55 • Increasing Binary Log Retention for Amazon RDS DB Instances • Log Message: Some changes from the source database had no impact when applied to the target database • Error: Identifier too long • Error: Unsupported Character Set Causes Field Data Conversion to Fail • Error: Codepage 1252 to UTF8 [120112] A field data conversion failed This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 50 CDC Task Failing for Amazon RDS DB Instance E ndpoint Because Binary Logging Disabled This issue occurs with Amazon RDS DB instances because automated backups are disabled Enable automatic backups by setting the backup retention period to a non  zero value Connections to a target MySQL instance are disconnected during a task If you have a task with LOBs that is getting disconnected from a MySQL target with the following type of errors in the task log you might need to adjust some of your task settings [TARGET_LOAD ]E: RetCode: SQL_ ERROR SqlState : 08S01 NativeError: 2013 Message: [ MySQL][ODBC 53(w) Driver ][mysqld5716log]Lost connection to MySQL server during query [122502] ODBC general error To solve the issue where a task is being disconnected from a MySQL target do the following: • Check that you have your database variable max_allowed_packet set large enough to hold your largest LOB • Check that you have the following variables set to have a large timeout value We suggest you use a value of at least 5 minutes for each of these variables  o net_read_timeout o net_write_timeout o wait_timeout o interactive_timeout Adding Autocommit to a MySQL compatible Endpoint To add autocommit to a target MySQL compatible endpoint  use the following procedure: This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 51 1 Sign in to the AWS Management Console and sel ect DMS  2 Select Endpoints  3 Select the MySQL compatible target endpoint that you want to add autocommit to 4 Select Modify  5 Select Advanced  and then add the following code to the Extra connection attributes text box: Initstmt = SET AUTOCOMMIT= 1 6 Choose Modify  Disable Foreign Keys on a Target MySQL compatible Endpoint You can disable foreign key checks on MySQL by adding the following to the Extra Connection Attributes in the Advanced section of the target MySQL Am azon Aurora with MySQL compatibility or MariaDB endpoint To disable foreign keys on a target MySQL compatible endpoint  use the following procedure: 1 Sign in to the AWS Management Console and select DMS  2 Select Endpoints  3 Select the MySQL Aurora MySQL  or MariaDB target endpoint that you want to disable foreign keys 4 Select Modify  5 Select Advanced  and then add the following code to the Extra connection attributes text box: Initstmt =SET FOREIGN_KEY_CHECKS= 0 This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 52 6 Choose Modify  Characters Replaced with Question Mark The most common situation that causes this issue is when the source endpoint characters have been encoded by a character set that AWS DMS doesn\\'t support For example AWS DMS engine versions prior to version 311 do n\\'t support the UTF8MB4 character set Bad event Log Entries Bad event entries in the migration logs usually indicate that an unsupported DDL operation was attempted on the source database endpoint Unsupported DDL operations cause an event that the repli cation instance cannot skip so a bad event is logged To fix this issue restart the task from the beginning which will reload the tables and will start capturing changes at a point after the unsupported DDL operation was issued Change Data Capture with MySQL 55 AWS DMS change data capture (CDC) for Amazon RDS MySQL compatible databases requires full image row based binary logging which is not supported in MySQL version 55 or lower To use AWS DMS CDC you must up upgrade your Amazon RDS DB instance t o MySQL version 56 Increasing Binary Log Retention for Amazon RDS DB Instances AWS DMS requires the retention of binary log files for change data capture To increase log retention on an Amazon RDS DB instance use the following procedure The following example increases the binary log retention to 24 hours call mysqlrds_set_confi guration( \\'binlog retention hours\\'  24); This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 53 Log Message: Some changes from the source database had no impact when applied to the target database When AWS DMS updates a MySQL database column’s value to its existing value a message of zero rows a ffected is returned from MySQL This behavior is unlike other database engines such as Oracle and SQL Server that perform an update of one row even when the replacing value is the same as the current one Error: Identifier too long The following error oc curs when an identifier is too long: TARGET_LOAD E: RetCode: SQL_ERROR SqlState: HY000 NativeError: 1059 Message: MySQLhttp://ODBC 53(w) Driverhttp://mysqld  5610Identifier name \\'<name>\\' is too long 122502 ODBC general error (ar_odbc_stmtc: 4054) When AWS DMS is set to create the tables and primary keys in the target database it currently does not use the same names for the Primary Keys that were used in the source database Instead AWS DMS creates the Primary Key na me based on the tables name When the table name is long the auto generated identifier created can be longer than the allowed limits for MySQL The solve this issue currently pre create the tables and Primary Keys in the target database and use a task w ith the task setting Target table preparation mode set to Do nothing or Truncate to populate the target tables Error: Unsupported Character Set Causes Field Data Conversion to Fail The following error occurs when an unsupported character set causes a fi eld data conversion to fail: \"[SOURCE_CAPTURE ]E: Column \\'<column name>\\' uses an unsupported character set [120112] A field data conversion failed (mysql_endpoint_capturec: 2154) This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 54 This error often occurs because of tables or databases using U TF8MB4 encoding AWS DMS engine versions prior to 311 don\\'t support the UTF8MB4 character set In addition check your database\\'s parameters related to connections The following command can be used to see these parameters: SHOW VARIABLES LIKE \\'%char%\\' ; Error: Codepage 1252 to UTF8 [120112] A field data conversion failed The following error can occur during a migration if you have non codepage 1252 characters in the source MySQL database [SOURCE_CAPTURE ]E: Error converting column \\'column_xyz\\' in tabl e \\'table_xyz with codepage 1252 to UTF8 [120112] A field data conversion failed (mysql_endpoint_capturec: 2248) As a workaround you can use the CharsetMapping extra connection attribute with your source MySQL endpoint to specify character set mapping You might need to restart the AWS DMS migration task from the beginning if you add this extra connection attribute For example the following extra connection a ttribute could be used for a MySQL source endpoint where the source character set is utf8 or latin1  65001 is the UTF8 code page identifier CharsetMapping =utf865001 CharsetMapping =latin165001 Conclusion Amazon Aurora is a high performance highly available and enterprise grade database built for the cloud Leveraging Amazon Aurora can result in better performance and greater availability than other open source databases and lower costs than most commercial grade databases This paper proposes stra tegies for identifying the best method to migrate databases to Amazon Aurora and details the procedures for planning This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 55 and executing those migrations In particular AWS Database Migration Service (AWS DMS) as well as the AWS Schema Conversion Tool are the r ecommended tools for heterogeneous migration scenarios These powerful tools can greatly reduce the cost and complexity of database migrations Multiple factors contribute to a successful database migration: • The choice of the database product • A migration approach (eg methods tools) that meets performance and uptime requirements • Welldefined migration procedures that enable database administrators to prepare test and complete all migration steps with confidence • The ability to identify diagnose and deal with issues with little or no interruption to the migration process We hope that the guidance provided in this document will help you introduce meaningful improvements in all of these areas and that it will ultimately contribute to creating a bette r overall experience for your database migrations into Amazon Aurora Contributors Contributors to this document include : • Bala Mugunthan Sr Partner Solution Architect Amazon Web Services • Ashar Abbas Database Specialty Architect • Sijie Han SA Manager A mazon Web Services • Szymon Komendera  Database Engineer Amazon Web Services This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 56 Further Reading For additional information  see: • Aurora on Amazon RDS User Guide • Migrating Your Databases t o Amazon Aurora AWS whitepaper • Best Practices for Migrating MySQL Databases to Amazon Aurora AWS whitepaper Document Revisions Date Description July 2020 Added information for the large databases migrations on Amazon Aurora and functional p artition and data shard consolidation strategies are discussed in homogenous migration s ection s Multi threaded migration using mydumper and myload er open source tools are introduced  Overall basic acceptance testing functional test non functional test and user acceptance tests are explained in the testing phase and pre cutover and post cut  overs phase scenarios are further explained  September 2019 First publication',\n",
              " 'Category': 'General'}"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Preview a document\n",
        "docs[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3XI6DDDUIrnT"
      },
      "source": [
        "# Step 4: Chunk up the data\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "PcyCLgTYIrnT"
      },
      "outputs": [],
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from typing import Dict, List"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "bbFxHwy3IrnT"
      },
      "outputs": [],
      "source": [
        "# Separators to split on\n",
        "separators = [\"\\n\\n\", \"\\n\", \" \", \"\", \"#\", \"##\", \"###\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sbo9ofvsIrnT"
      },
      "source": [
        "📚 https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/split_by_token/#tiktoken\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "fBTkL7msIrnU"
      },
      "outputs": [],
      "source": [
        "# Use the `RecursiveCharacterTextSplitter` text splitter with the `cl100k_base` encoding\n",
        "# For text data, you typically want to keep 1-2 paragraphs (~200 tokens) in a single chunk\n",
        "# Chunk overlap of 15-20% of the chunk size is recommended\n",
        "# Pass the `separators` list above as an argument called `separators`\n",
        "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
        "    encoding_name=\"cl100k_base\", separators=separators, chunk_size=200, chunk_overlap=30\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hqbZApbtIrnU"
      },
      "source": [
        "📚 https://api.python.langchain.com/en/latest/character/langchain_text_splitters.character.RecursiveCharacterTextSplitter.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "90qk-pJDIrnU"
      },
      "outputs": [],
      "source": [
        "def get_chunks(doc: Dict, text_field: str) -> List[Dict]:\n",
        "    \"\"\"\n",
        "    Chunk up a document.\n",
        "\n",
        "    Args:\n",
        "        doc (Dict): Parent document to generate chunks from.\n",
        "        text_field (str): Text field to chunk.\n",
        "\n",
        "    Returns:\n",
        "        List[Dict]: List of chunked documents.\n",
        "    \"\"\"\n",
        "    # Extract the field to chunk from `doc`\n",
        "    text = doc[text_field]\n",
        "    # Split `text` using the appropriate method of the `RecursiveCharacterTextSplitter` class\n",
        "    # NOTE: `text` is a string\n",
        "    chunks = text_splitter.split_text(text)\n",
        "\n",
        "    # Iterate through `chunks` and for each chunk:\n",
        "    # 1. Create a shallow copy of `doc`, call it `temp`\n",
        "    # 2. Set the `text_field` field in `temp` to the content of the chunk\n",
        "    # 3. Append `temp` to `chunked_data`\n",
        "    chunked_data = []\n",
        "    for chunk in chunks:\n",
        "       temp = doc.copy()\n",
        "       temp[text_field]=chunk\n",
        "       chunked_data.append(temp)\n",
        "\n",
        "    return chunked_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "3xCXAXqOIrnU"
      },
      "outputs": [],
      "source": [
        "split_docs = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "xYn8geANIrnU"
      },
      "outputs": [],
      "source": [
        "# Iterate through `docs`, use the `get_chunks` function to chunk up the documents based on the \"body\" field, and add the list of chunked documents to `split_docs` initialized above.\n",
        "split_docs = []\n",
        "for doc in docs:\n",
        "    chunks = get_chunks(doc, \"Content\")\n",
        "    split_docs.extend(chunks)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mEFDkVdqIrnU",
        "outputId": "e7a76c1d-04f7-443c-9cd0-fe9372f33907"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "970"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check that the length of the list of chunked documents is greater than the length of `docs`\n",
        "len(split_docs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_PW8NAHsIrnU",
        "outputId": "5b5a2273-c376-435b-d845-b1ee201dc925"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'Title': 'Amazon_Aurora_Migration_Handbook',\n",
              " 'Content': 'This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 1 Amazon Aurora Migration Handbook July 2020 This paper has been archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 2 Notices Customers are responsible for making their own independent assessment of the information in this document This document: (a) is for informational purposes only (b) represents current AWS product offerings and practices which are subject to change withou t notice and (c) does not create any commitments or assurances from AWS and its affiliates suppliers or licensors AWS products or services are provided “as is” without warranties representations or conditions of any kind whether express or implied The responsibilities and liabilities of',\n",
              " 'Category': 'General'}"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Preview one of the items in split_docs- ensure that it is a Python dictionary\n",
        "split_docs[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjDFGqVYIrnU"
      },
      "source": [
        "# Step 5: Generate embeddings\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87,
          "referenced_widgets": [
            "deb411770ca14acea2132206fcb54bd1",
            "cc7ace7ad1644083bd7362cb03795dea",
            "7758f0d9087749eda61f44458096985d",
            "fb20472b5c944892933653f4fd1e0694",
            "65717255d10a4d3a832f916ed177d081",
            "cce96eb1a33a4480b933e5211397be9e",
            "2ea0d6344ef24f0e9de5cb7d1951ea2d",
            "20c82bc105d64affbbdd7a43a96441e1",
            "f3fcd4b0f25f46f88e37abea32725ac1",
            "7a6139f6ae86449aae7a95aec0d5d3f6",
            "9c435b51c3624a3e89ec9712ddb6e523"
          ]
        },
        "id": "HbJJ1xxmIrnU",
        "outputId": "cf0ddb4f-273a-4406-f8c0-4c4860d22571"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "deb411770ca14acea2132206fcb54bd1",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "from tqdm import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "4hM8qCh1IrnU"
      },
      "outputs": [],
      "source": [
        "# Load the `gte-small` model using the Sentence Transformers library\n",
        "\n",
        "import json\n",
        "import boto3\n",
        "from botocore.config import Config\n",
        "\n",
        "my_config = Config(\n",
        "    region_name = 'us-east-1',\n",
        "    signature_version = 'v4',\n",
        "    retries = {\n",
        "        'max_attempts': 10,\n",
        "        'mode': 'standard'\n",
        "    }\n",
        ")\n",
        "\n",
        "class TitanEmbeddings(object):\n",
        "    accept = \"application/json\"\n",
        "    content_type = \"application/json\"\n",
        "\n",
        "    def __init__(self, model_id=\"amazon.titan-embed-text-v2:0\"):\n",
        "        self.bedrock = boto3.client(\n",
        "            service_name='bedrock-runtime',\n",
        "            config=my_config,\n",
        "            aws_access_key_id=os.environ[\"AWS_ACCESS_KEY\"],\n",
        "            aws_secret_access_key=os.environ[\"AWS_SECRET\"],\n",
        "        )\n",
        "        self.model_id = model_id\n",
        "    def __call__(self, text, dimensions, normalize=True):\n",
        "        \"\"\"\n",
        "        Returns Titan Embeddings\n",
        "        Args:\n",
        "            text (str): text to embed\n",
        "            dimensions (int): Number of output dimensions.\n",
        "            normalize (bool): Whether to return the normalized embedding or not.\n",
        "        Return:\n",
        "            List[float]: Embedding\n",
        "\n",
        "        \"\"\"\n",
        "        body = json.dumps({\n",
        "            \"inputText\": text,\n",
        "            \"dimensions\": dimensions,\n",
        "            \"normalize\": normalize\n",
        "        })\n",
        "        response = self.bedrock.invoke_model(\n",
        "            body=body, modelId=self.model_id, accept=self.accept, contentType=self.content_type\n",
        "        )\n",
        "        response_body = json.loads(response.get('body').read())\n",
        "        return response_body['embedding']\n",
        "\n",
        "embedding_model = TitanEmbeddings(model_id=\"amazon.titan-embed-text-v2:0\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j0NcDhr7IrnU"
      },
      "source": [
        "📚 https://huggingface.co/thenlper/gte-small"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "B8qwJuPDIrnU"
      },
      "outputs": [],
      "source": [
        "# Define a function that takes a piece of text (`text`) as input, embeds it using the `embedding_model` instantiated above and returns the embedding as a list\n",
        "# An array can be converted to a list using the `tolist()` method\n",
        "def get_embedding(text: str) -> List[float]:\n",
        "    \"\"\"\n",
        "    Generate the embedding for a piece of text.\n",
        "\n",
        "    Args:\n",
        "        text (str): Text to embed.\n",
        "\n",
        "    Returns:\n",
        "        List[float]: Embedding of the text as a list.\n",
        "    \"\"\"\n",
        "    dimensions = 1024\n",
        "    normalize = True\n",
        "    embedding = embedding_model(text, dimensions, normalize)\n",
        "\n",
        "    return embedding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "l83X8RBzIrnU"
      },
      "outputs": [],
      "source": [
        "embedded_docs = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D76T0acEIrnU",
        "outputId": "114eb5da-2dbb-4b5a-cd83-7f954156f72c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 970/970 [04:07<00:00,  3.93it/s]\n"
          ]
        }
      ],
      "source": [
        "# Add an `embedding` field to each dictionary in `split_docs`\n",
        "# The `embedding` field should correspond to the embedding of the value of the `body` field\n",
        "# Use the `get_embedding` function defined above to generate the embedding\n",
        "# Append the updated dictionaries to `embedded_docs` initialized above.\n",
        "for doc in tqdm(split_docs):\n",
        "    doc[\"embedding\"] = get_embedding(doc[\"Content\"])\n",
        "\n",
        "    embedded_docs.append(doc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZzGSNpPDIrnU",
        "outputId": "0080f1c5-e5b3-478d-b7f3-273488e2fd69"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "970"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check that the length of `embedded_docs` is the same as that of `split_docs`\n",
        "len(embedded_docs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "31HVf4viIrnU"
      },
      "source": [
        "# Step 6: Ingest data into MongoDB\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "E_29fQtmIrnU"
      },
      "outputs": [],
      "source": [
        "# Name of the database -- Change if needed or leave as is\n",
        "DB_NAME = \"mongodb_rag_lab\"\n",
        "# Name of the collection -- Change if needed or leave as is\n",
        "COLLECTION_NAME = \"knowledge_base\"\n",
        "# Name of the vector search index -- Change if needed or leave as is\n",
        "ATLAS_VECTOR_SEARCH_INDEX_NAME = \"vector_index\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OmKDq5i4IrnU"
      },
      "source": [
        "📚 https://pymongo.readthedocs.io/en/stable/tutorial.html#getting-a-database\n",
        "\n",
        "📚 https://pymongo.readthedocs.io/en/stable/tutorial.html#getting-a-collection"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "sKrZumaEIrnU"
      },
      "outputs": [],
      "source": [
        "# Connect to the collection defined above using the `mongodb_client` defined in Step 2\n",
        "collection = mongodb_client[DB_NAME][COLLECTION_NAME]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oQMTwoirIrnV",
        "outputId": "2b8ee829-ef4c-49ca-8144-a5a35fa8d01c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DeleteResult({'n': 0, 'electionId': ObjectId('7fffffff000000000000001f'), 'opTime': {'ts': Timestamp(1727162815, 10), 't': 31}, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1727162815, 10), 'signature': {'hash': b'\\xdd\\x82\\x1f\\xc3\\x91s\\xa9\\xe5\\xf7\\xde4\\xcd\\xc2N\\xc9g\\x1by\\xe4\\xdc', 'keyId': 7384910504117927940}}, 'operationTime': Timestamp(1727162815, 10)}, acknowledged=True)"
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Bulk delete all existing records from the collection defined above\n",
        "collection.delete_many({})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uD8OKgQTIrnV"
      },
      "source": [
        "📚 https://pymongo.readthedocs.io/en/stable/examples/bulk.html#bulk-insert\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpkhiDr3IrnV",
        "outputId": "f57f852b-80bf-4b4e-9b3b-4e0578fd8fe7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Data ingestion into MongoDB completed\n"
          ]
        }
      ],
      "source": [
        "# Bulk insert `embedded_docs` into the collection defined above -- should be a one-liner\n",
        "collection.insert_many(embedded_docs)\n",
        "\n",
        "print(\"Data ingestion into MongoDB completed\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JZiff8s8IrnX"
      },
      "source": [
        "# Step 7: Create a vector search index\n",
        "\n",
        "Follow the instructions in the documentation to create a Vector Search index in the Atlas UI.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FF4jw2APIrnX"
      },
      "source": [
        "# Step 8: Perform semantic search on your data\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_aNpEvbeIrnY"
      },
      "source": [
        "### Define a vector search function\n",
        "\n",
        "📚 https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#fields\n",
        "\n",
        "📚 https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#ann-examples (Refer to the \"Basic Example\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "SFBReDiuIrnY"
      },
      "outputs": [],
      "source": [
        "# Define a function to retrieve relevant documents for a user query using vector search\n",
        "def vector_search(user_query: str) -> List[Dict]:\n",
        "    \"\"\"\n",
        "    Retrieve relevant documents for a user query using vector search.\n",
        "\n",
        "    Args:\n",
        "    user_query (str): The user's query string.\n",
        "\n",
        "    Returns:\n",
        "    list: A list of matching documents.\n",
        "    \"\"\"\n",
        "\n",
        "    # Generate embedding for the `user_query` using the `get_embedding` function defined in Step 5\n",
        "    query_embedding = get_embedding(user_query)\n",
        "\n",
        "    # Define an aggregation pipeline consisting of a $vectorSearch stage, followed by a $project stage\n",
        "    # Set the number of candidates to 150 and only return the top 5 documents from the vector search\n",
        "    # In the $project stage, exclude the `_id` field and include only the `body` field and `vectorSearchScore`\n",
        "    # NOTE: Use variables defined previously for the `index`, `queryVector` and `path` fields in the $vectorSearch stage\n",
        "    pipeline = [\n",
        "      {\n",
        "          \"$vectorSearch\": {\n",
        "              \"index\": ATLAS_VECTOR_SEARCH_INDEX_NAME,\n",
        "              \"queryVector\": query_embedding,\n",
        "              \"path\": \"embedding\",\n",
        "              \"numCandidates\": 150,\n",
        "              \"limit\": 5,\n",
        "          }\n",
        "      },\n",
        "      {\n",
        "          \"$project\": {\n",
        "              \"_id\": 0,\n",
        "              \"Content\": 1,\n",
        "              \"score\": {\"$meta\": \"vectorSearchScore\"}\n",
        "          }\n",
        "      }\n",
        "  ]\n",
        "\n",
        "    # Execute the aggregation `pipeline` and store the results in `results`\n",
        "    results = collection.aggregate(pipeline)\n",
        "\n",
        "    return list(results)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8769kz1LIrnY"
      },
      "source": [
        "### Run vector search queries\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6SFeBGAtIrnY",
        "outputId": "620205cc-5bd5-44f6-e740-2f48643a07ca"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'Content': 'For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 31 Notes For the sake of simplicity this scenario assumes the following: 1 Migration commands are executed from a client instance running a Linux operating system 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) that is configured to allow connections from the client instance 3 The target Aurora DB cluster already exists and is configured to allow connections from the client instance If you don’t yet have an Aurora DB cluster review the stepbystep cluster launch instructions in the Amazon RDS User Guide 17 4 Export from the source database is performed using a privileged super  user MySQL ac count For simplicity this scenario assumes that the user holds all permissions available in MySQL 5 Import into Amazon Aurora is performed using the Aurora master user account that is the',\n",
              "  'score': 0.8689587712287903},\n",
              " {'Content': 'For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 31 Notes For the sake of simplicity this scenario assumes the following: 1 Migration commands are executed from a client instance running a Linux operating system 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) that is configured to allow connections from the client instance 3 The target Aurora DB cluster already exists and is configured to allow connections from the client instance If you don’t yet have an Aurora DB cluster review the stepbystep cluster launch instructions in the Amazon RDS User Guide 17 4 Export from the source database is performed using a privileged super  user MySQL ac count For simplicity this scenario assumes that the user holds all permissions available in MySQL 5 Import into Amazon Aurora is performed using the Aurora master user account that is the',\n",
              "  'score': 0.8689587712287903},\n",
              " {'Content': 'your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 41 1 You execute the migration commands from client instances running a Linux operating system: a Client instance A is located in the source server’s network b Clien t instance B is located in the same Amazon VPC Availability Zone and Subnet as the target Aurora cluster 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) configured to allow connections from client instance A 3 The target Aurora DB cluster already exists and is configured to allow connections from client instance B If you don’t have an Aurora DB cluster yet review the stepbystep cluster launch instructions in the Amazon RDS User Guide  4 Communication is allowed between both client instances 5 You perform the export from the source database using a privileged super user MySQL account For simplicity the example assumes that the user holds all permissions available in MySQL 6 You perform the import into Amazon Aurora using the master user account that',\n",
              "  'score': 0.8682036995887756},\n",
              " {'Content': 'your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 41 1 You execute the migration commands from client instances running a Linux operating system: a Client instance A is located in the source server’s network b Clien t instance B is located in the same Amazon VPC Availability Zone and Subnet as the target Aurora cluster 2 The source server is a self managed MySQL database (eg running on Amazon EC2 or on premises) configured to allow connections from client instance A 3 The target Aurora DB cluster already exists and is configured to allow connections from client instance B If you don’t have an Aurora DB cluster yet review the stepbystep cluster launch instructions in the Amazon RDS User Guide  4 Communication is allowed between both client instances 5 You perform the export from the source database using a privileged super user MySQL account For simplicity the example assumes that the user holds all permissions available in MySQL 6 You perform the import into Amazon Aurora using the master user account that',\n",
              "  'score': 0.8682036995887756},\n",
              " {'Content': 'archived For the latest Ama zon Aurora Migration content refer to: https://d1awsstaticcom/whitepapers/RDS/Migrating your databases to Amazon Aurorapdf Amazon Web Services Amazon Aurora Migration Handbook 42 The utilities depend on several packages that you should install first [ec2user@clientA ~]$ sudo yum install glib2 devel mysql56 \\\\ mysql56devel zlib devel pcre devel openssl devel g++ gcc c++ cmake The next steps involve creating a directory to hold the program sources and then fetching and unpacking the source archive [ec2user@clientA ~]$ mkdir mydumper [ec2 user@clientA ~]$ cd mydumper/ [ec2user@clientA mydumper]$ wget https://launchp adnet/mydumper/09/091/+download/mydumper  091targz 20160629 21:39:03 (153 KB/s)  ‘mydumper',\n",
              "  'score': 0.8582425117492676}]"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "vector_search(\"How to migrate Aurora to other platform?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PQFZnfOsIrnY",
        "outputId": "37415411-0779-4af2-96ee-f80b821ef6a3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'Content': 'a concentration of intellectual capital and innovative businesses and startups can be a strong indicator of economic development Cloud technology can help give new businesses a boost in their forecasting demand generation and innovation when bringing their products or services to market AWS accelerate s this process through AWS Activate  a program designed to provide startups with resourc es and credits to get started with the cloud; through access to tools like Amazon LightSail  which provides technology like virtual private servers to enterprises of all sizes for the cost of a cup of ArchivedAmazon Web Services  Inc – 5 Ways t he Cloud Can Drive Economi c Development Page 7 coffee ; and by encouraging public private partnerships and small business linkages namely through the strength of the AWS Partner Network (APN)  Additionally AWS CloudStart formed to encourage the growth of SMEs and economic development organizations by providing resources to educate train and help these entities embrace the cost effectiveness of the AWS Cloud “As small businesses leverage a broader portfolio of digital solutions they can see an increase in',\n",
              "  'score': 0.8080956339836121},\n",
              " {'Content': 'Archived5 Ways the Cloud Can Drive Economic Development August 2018 This paper has been archived For the latest technical content see the AWS Whitepapers & Guides page: https://awsamazoncom/whitepapersArchived © 201 8 Amazon Web Services Inc or its affiliates All rights reserved Notices This document is provided for informational purposes only It represents Amazon Web Services’s (“AWS”) current product offerings and practices as of the date of issue of this document which are subject to change without notice Customers are responsible for making their own independent assessment of the information in this docu ment and any use of AWS’s products or services each of which is provided “as is” without warranty of any kind whether express or implied This document does not create any warranties representations contractual commitments conditions or assurances fro m AWS its affiliates suppliers or licensors The responsibilities and liabilities of AWS to its customers are controlled by AWS agreements and this document is not part of nor does it modify any agreement between AWS and its customers Archived Contents Introduction 1 Sharing',\n",
              "  'score': 0.7892063856124878},\n",
              " {'Content': 'applications to AWS to simplify infrastructure management deploy quicker provide greater availability increase agility allow for faster innovation and lower cost Having a clear understanding of existing infrastructure costs the components of your migration bubble and their corresponding costs and projected savings will help you calculate payback time and projected ROI With a long history in enabling enterprises to successfully adopt cloud computing Amazon Web Services delivers a mature set of services specifically designed for the unique security compliance privacy and governance requirements of large organizations With a technology platform that is both broad and deep Professional Services and Support organizations robust training programs and an ecosystem tens ofthousands strong AWS can help you move faster and do more With AWS you can: \\uf095 Take advantage of more services storage options and security controls than any other cloud platform \\uf095 Deliver on stringent standards with the broadest set of certifications accreditations and controls in the industry \\uf095 Get deep assistance with our global cloud focused enterprise professional services support and training teams ArchivedAmazon Web Services – A Practical Guide to Cloud Migration December 2015 Page 13',\n",
              "  'score': 0.7749395370483398},\n",
              " {'Content': 'AWS  It also discuss es the benefits of running Oracle E Business suite on AWS  and various use cases AWS overview AWS provides on demand computing resources and services in the cloud with pay as yougo pricing As of the date of this publication AWS serves over a million active customers in more than 190 countries  and is available in 25 AWS Regions worldwide You can run a server on AWS and log in configure secure and operate it just as you would operate a server in your own data center Using AWS resources for your compute needs is like purchasing electricity from a power company instead of runn ing your own generator and it provides many of the same benefits: • The capacity you get exactly matches your needs • You pay only for what you use • Economies of scale result in lower costs • The service is provided by a vendor who is experienced in running l argescale compute and network systems This version has been archived For the latest version of this document visit:',\n",
              "  'score': 0.7682077884674072},\n",
              " {'Content': 'innovation ArchivedAmazon Web Services  Inc – 5 Ways t he Cloud Can Drive Economi c Development Page 8 A municipality in Sweden is feeling the effects of a strategic partnership aimed at helping small bu sinesses adapt and thrive  Consultant CAG Malardalen in Västerås Sweden  uses the cloud to help constituents make more data driven decisions deploy resources more efficiently  and help shape the economic conditions essential for attract ing new economic activity  “[We are] striving to bring the region the latest in cloud technology Our ambition is to always deliver the most relevant IT solutions to our customers Through working with AWS CloudStart our customers benefit from the foundational knowledge we have gathered and we are already seeing a lot of new possibilities for us as a service provider across Sweden” says Tomas Täuber CEO of CAG Malardalen Allocating Resources More Effectively Cloud technology allows governments to rethink critical processes  It builds new efficiencies across procurement security compliance and data protection Additionally the cost effectiveness of the cloud enables',\n",
              "  'score': 0.7669878005981445}]"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "vector_search(\"What industry or development can AWS cloud benefit?\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGp-Vy9QIrnY"
      },
      "source": [
        "# 🦹‍♀️ Combine pre-filtering with vector search\n",
        "\n",
        "📚 https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-type/#about-the-filter-type\n",
        "\n",
        "📚 https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#ann-examples (Refer to the \"Filter Example\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XAsYf3pLIrnY"
      },
      "source": [
        "# Step 9: Build the RAG application\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m3NBJjBZIrnY"
      },
      "source": [
        "### Instantiate a chat model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "alp3AcQXIrnZ"
      },
      "source": [
        "### Define a function to create the chat prompt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "id": "wwjE0OhHIrnZ"
      },
      "outputs": [],
      "source": [
        "# Define a function to create the user prompt for our RAG application\n",
        "def create_prompt(user_query: str) -> str:\n",
        "    \"\"\"\n",
        "    Create a chat prompt that includes the user query and retrieved context.\n",
        "\n",
        "    Args:\n",
        "        user_query (str): The user's query string.\n",
        "\n",
        "    Returns:\n",
        "        str: The chat prompt string.\n",
        "    \"\"\"\n",
        "    # Retrieve the most relevant documents for the `user_query` using the `vector_search` function defined in Step 8\n",
        "    context = vector_search(user_query)\n",
        "\n",
        "    # Join the retrieved documents into a single string, where each document is separated by two new lines (\"\\n\\n\")\n",
        "    context = \"\\n\\n\".join([doc.get('Content') for doc in context])\n",
        "    # Prompt consisting of the question and relevant context to answer it\n",
        "    prompt = f\"Answer the question based only on the following context. If the context is empty, say I DON'T KNOW\\n\\nContext:\\n{context}\\n\\nQuestion:{user_query}\"\n",
        "    return prompt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PNBSDmh0IrnZ"
      },
      "source": [
        "### Define a function to answer user queries\n",
        "\n",
        "📚 https://docs.fireworks.ai/guides/querying-text-models#chat-completions-api"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "id": "B8qfr4ZmIrnZ"
      },
      "outputs": [],
      "source": [
        "# Define a function to answer user queries using Fireworks' Chat Completion API\n",
        "def generate_answer(user_query: str) -> None:\n",
        "    \"\"\"\n",
        "    Generate an answer to the user query.\n",
        "\n",
        "    Args:\n",
        "        user_query (str): The user's query string.\n",
        "    \"\"\"\n",
        "    client = boto3.client(\n",
        "            service_name='bedrock-runtime',\n",
        "            config=my_config,\n",
        "            aws_access_key_id=os.environ[\"AWS_ACCESS_KEY\"],\n",
        "            aws_secret_access_key=os.environ[\"AWS_SECRET\"],\n",
        "    )\n",
        "\n",
        "    # Set the model ID, e.g., Titan Text Premier.\n",
        "    model_id = \"amazon.titan-text-premier-v1:0\"\n",
        "\n",
        "    # Use the `create_prompt` function above to create a chat prompt\n",
        "    prompt = create_prompt(user_query)\n",
        "\n",
        "    conversation = [\n",
        "          {\n",
        "              \"role\": \"user\",\n",
        "              \"content\": [{\"text\": prompt}],\n",
        "          }\n",
        "      ]\n",
        "\n",
        "    # Use the `prompt` created above to populate the `content` field in the chat message\n",
        "    response = client.converse(\n",
        "        modelId=model_id,\n",
        "        messages=conversation,\n",
        "        inferenceConfig={\"topP\":0.9},\n",
        "        additionalModelRequestFields={}\n",
        "    )\n",
        "\n",
        "    # Print the final answer\n",
        "    print(response[\"output\"][\"message\"][\"content\"][0][\"text\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4gtuF62IrnZ"
      },
      "source": [
        "### Query the RAG application\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b34B2S1zIrnZ",
        "outputId": "8066eb1d-72b9-4530-e65d-8f4d6b15cd5a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloud technology can help give new businesses a boost in their forecasting demand generation and innovation when bringing their products or services to market\n"
          ]
        }
      ],
      "source": [
        "generate_answer(\"What industry or development can AWS cloud benefit?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "auxo3K_rIrnZ",
        "outputId": "12df1d3e-00dc-4628-aa52-d843135fd7ad"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "I DON'T KNOW\n"
          ]
        }
      ],
      "source": [
        "generate_answer(\"What did I just ask you?\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lCXeOBLTIrnZ"
      },
      "source": [
        "# 🦹‍♀️ Re-rank retrieved results\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "id": "5iQySEQxIrnZ"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import CrossEncoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241,
          "referenced_widgets": [
            "aa3f5fe0a6df4288a833a6dc9c7d839e",
            "278c4e3d670f4fb4a04117ce3508311f",
            "ccaee1a3af4743f9a441170180db3a5f",
            "46354ca3ee7d4600b78420887a1dde2c",
            "b25684ada2ca4c3ba38623e75ffa6e27",
            "6c8c6eabbd1a4e3c9239dd45ba2fa776",
            "6f965b1a2cc349fbaebf1a50f0a40aee",
            "c2f8658ef30849c8b70e362b79766309",
            "2300400ebe964c4babb749d995e05e5a",
            "f517e78e15d14aa5b37c04d367c99804",
            "4e1be448f7494f7eb5b8e09504e37362",
            "85e3db77cb7e46718689089639713b10",
            "d5b227dedc4346998f88260b04d0600c",
            "2c2d988b2b8c472dbc0dd75385d7c48f",
            "585f4c87521849288f6ceff6fb7920a8",
            "41e1666413ac433688680c53a437180f",
            "c71afb2162b94738bbf93b1976097a08",
            "035a50ce208a45258d5b212e8466343f",
            "9d36c7f3f08b4dfe95e5bc945f3e1208",
            "de0822612c0f43ebb097ebaf557216f4",
            "c269701f862549b2b6a91d8ab5838b08",
            "1d8c4f8d764344e8817dee2228cf6dfd",
            "eb00279c0176456db4a44802acefbaa0",
            "e115fd45185441888a6e45d00f9e0260",
            "7c7df8af99004089800e6892af658576",
            "fa72acef332e48b5948c69cf2e9b2573",
            "d429e62af04a4becbb784ad6041221a4",
            "3b958bb934ad4a089f135e49e4964a95",
            "5213f7c794b940a392ecf788cfc34869",
            "611f7ff8cb5d4bffa8b37fb7f33a67e5",
            "9fadd4111cb24606942c68816e2450f5",
            "a8e895a8ac6c4b31ac27401e8b59969b",
            "d5a4f9f8c31842b2af82a15670d9dc6e",
            "5e7ea3dec2d84befbfeca81e59887e37",
            "595fb968931940d58025ca940fb51511",
            "8a7b2d1688bc49f49fb2dc5feee1aa3b",
            "cbd6067529b44889b050766db92a9c11",
            "5f161fc8634148b3bd6c6f576cc5b54c",
            "5999315d452b463db818093d62f0c37c",
            "5debc9238ea2428e949d3596503367eb",
            "a221234962054cb2b29ff66abe258d5c",
            "40a21479b8194121960325bf655bb5ce",
            "cc23f068cf1e489fb9254a128290250e",
            "5361e8c2daf24b60bc25e2e2f8df9bf2",
            "8c62eeeed4844094873a379ec87a778c",
            "7a11f8bed3e54e718f12a778ce78088b",
            "2841d3f13e0a46beb272972e1fa37cec",
            "3a0ceaf134d64791bb1e979101658534",
            "b8ffdb1a268a4c33aa07ff14c6d32e49",
            "3d38dbf5f34d48d6b0c71e9927b36604",
            "375fee84502b4d10885db49b5d5a359f",
            "08f7a1a01a7043979ad29eac8a0e3d55",
            "0b80749f01934a70bcd8a62def2f1825",
            "4d2131547e4d45e18b6e81fc70e08e91",
            "9493053a1712497c98b3489b9fca0c5a",
            "19e9a3a1e78243d082e43f9835839f92",
            "5458a2b4995248e6a39eed0b4a0226aa",
            "449608fbc4804c90bd58f29a281b7619",
            "56b69c3ef27945cfa572afa1ed0741df",
            "d6ade1b8e5ac461da386c9321e77eff4",
            "b4c705996c2c4c2ca2fc0ce6ffb0b210",
            "85aacf8449a945e495b6af803c6b694b",
            "293e0c5d0f3e407da3e479be176f017e",
            "d51aafbee3e349bb9d28edb80aa83bd6",
            "147809ed82c7447483165638f21db1bd",
            "c72f3aeccd9d4e9d95935890e5e77155",
            "6e8ac2a6989f4ee49f86fca7cca3dab6",
            "da2e289c84db462194101f8ff02c6e2c",
            "c987709a64204d99bc959a44ad27332e",
            "02b591812552444fa6d543779a614097",
            "0d7fc927d9994ec28b1a4bd13c818f20",
            "3f00bb98d50f43e39ddb830c3da4a352",
            "280e6088dd084d2cba58a674710c217a",
            "2b74147cb6d5449bb63ed5710509b531",
            "37220c85e05c43ccbb114780c027a4e7",
            "8eaed0ab90eb41fa9bdbda63ba6c224e",
            "3bc2d9c5f0b240c9a92d780aa0dad2e5"
          ]
        },
        "id": "HSblx5K0IrnZ",
        "outputId": "68a78639-85f8-43e2-ff6a-552acd7bf427"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aa3f5fe0a6df4288a833a6dc9c7d839e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "config.json:   0%|          | 0.00/968 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "85e3db77cb7e46718689089639713b10",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/142M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eb00279c0176456db4a44802acefbaa0",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.45k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5e7ea3dec2d84befbfeca81e59887e37",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "spm.model:   0%|          | 0.00/2.46M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c62eeeed4844094873a379ec87a778c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/8.65M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "19e9a3a1e78243d082e43f9835839f92",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/23.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6e8ac2a6989f4ee49f86fca7cca3dab6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/970 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "rerank_model = CrossEncoder(\"mixedbread-ai/mxbai-rerank-xsmall-v1\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gYiKvWUMIrnZ"
      },
      "source": [
        "📚 https://huggingface.co/mixedbread-ai/mxbai-rerank-xsmall-v1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "id": "ml6LwqzXIrnZ"
      },
      "outputs": [],
      "source": [
        "# Add a re-ranking step to the following function\n",
        "def create_prompt(user_query: str) -> str:\n",
        "    \"\"\"\n",
        "    Create a chat prompt that includes the user query and retrieved context.\n",
        "\n",
        "    Args:\n",
        "        user_query (str): The user's query string.\n",
        "\n",
        "    Returns:\n",
        "        str: The chat prompt string.\n",
        "    \"\"\"\n",
        "    # Retrieve the most relevant documents for the `user_query` using the `vector_search` function defined in Step 8\n",
        "    context = vector_search(user_query)\n",
        "    # Extract the \"body\" field from each document in `context`\n",
        "    documents = [d.get(\"Content\") for d in context]\n",
        "    # Use the `rerank_model` instantiated above to re-rank `documents`\n",
        "    # Set the `top_k` argument to 5\n",
        "    reranked_documents = rerank_model.rank(\n",
        "        user_query, documents, return_documents=True, top_k=5\n",
        "    )\n",
        "    # Join the re-ranked documents into a single string, where each document is separated by two new lines (\"\\n\\n\")\n",
        "    context = \"\\n\\n\".join([d.get(\"text\", \"\") for d in reranked_documents])\n",
        "    # Prompt consisting of the question and relevant context to answer it\n",
        "    prompt = f\"Answer the question based only on the following context. If the context is empty, say I DON'T KNOW\\n\\nContext:\\n{context}\\n\\nQuestion:{user_query}\"\n",
        "    return prompt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bwTffrC4IrnZ",
        "outputId": "892941ec-b501-4dba-cf82-2fafa01e8319"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "innovative businesses and startups\n"
          ]
        }
      ],
      "source": [
        "# Note the impact of re-ranking on the generated answer\n",
        "generate_answer(\"What industry or development can AWS cloud benefit?\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1TrU1QPeIrnZ"
      },
      "source": [
        "# Step 10: Add memory to the RAG application\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "id": "Tu9wdUBYIrnZ"
      },
      "outputs": [],
      "source": [
        "from datetime import datetime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {
        "id": "Nj8CFmmFIrna"
      },
      "outputs": [],
      "source": [
        "history_collection = mongodb_client[DB_NAME][\"chat_history\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NbgnZqY6Irna"
      },
      "source": [
        "📚 https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.create_index\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1GFcTaSgIrna",
        "outputId": "2046ea3f-0cd9-4282-89c4-493ccdb70b65"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'session_id_1'"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Create an index on the key `session_id` for the `history_collection` collection\n",
        "history_collection.create_index(\"session_id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FoZKu7COIrna"
      },
      "source": [
        "### Define a function to store chat messages in MongoDB\n",
        "\n",
        "📚 https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.insert_one"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {
        "id": "32-A6ISZIrna"
      },
      "outputs": [],
      "source": [
        "def store_chat_message(session_id: str, role: str, content: str) -> None:\n",
        "    \"\"\"\n",
        "    Store a chat message in a MongoDB collection.\n",
        "\n",
        "    Args:\n",
        "        session_id (str): Session ID of the message.\n",
        "        role (str): Role for the message. One of `system`, `user` or `assistant`.\n",
        "        content (str): Content of the message.\n",
        "    \"\"\"\n",
        "    # Create a message object with `session_id`, `role`, `content` and `timestamp` fields\n",
        "    # `timestamp` should be set the current timestamp\n",
        "    message = {\n",
        "        \"session_id\": session_id,\n",
        "        \"role\": role,\n",
        "        \"content\": content,\n",
        "        \"timestamp\": datetime.now(),\n",
        "    }\n",
        "    # Insert the `message` into the `history_collection` collection\n",
        "    history_collection.insert_one(message)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uxf593MVIrna"
      },
      "source": [
        "### Define a function to retrieve chat history from MongoDB\n",
        "\n",
        "📚 https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.find\n",
        "\n",
        "📚 https://pymongo.readthedocs.io/en/stable/api/pymongo/cursor.html#pymongo.cursor.Cursor.sort"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 82,
      "metadata": {
        "id": "C7Sr374HIrna"
      },
      "outputs": [],
      "source": [
        "def retrieve_session_history(session_id: str) -> List:\n",
        "    \"\"\"\n",
        "    Retrieve chat message history for a particular session.\n",
        "\n",
        "    Args:\n",
        "        session_id (str): Session ID to retrieve chat message history for.\n",
        "\n",
        "    Returns:\n",
        "        List: List of chat messages.\n",
        "    \"\"\"\n",
        "    # Query the `history_collection` collection for documents where the \"session_id\" field has the value of the input `session_id`\n",
        "    # Sort the results in increasing order of the values in `timestamp` field\n",
        "    cursor =  history_collection.find({\"session_id\": session_id}).sort(\"timestamp\", 1)\n",
        "\n",
        "    if cursor:\n",
        "        # Iterate through the cursor and extract the `role` and `content` field from each entry\n",
        "        # Then format each entry as: {\"role\": <role_value>, \"content\": <content_value>}\n",
        "        messages = [{\"role\": msg[\"role\"], \"content\": [{ \"text\": msg[\"content\"]}]} for msg in cursor]\n",
        "    else:\n",
        "        # If cursor is empty, return an empty list\n",
        "        messages = []\n",
        "\n",
        "    return messages"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Yt7zJQ_Irna"
      },
      "source": [
        "### Handle chat history in the `generate_answer` function\n",
        "\n",
        "📚 https://docs.fireworks.ai/guides/querying-text-models#chat-completions-api\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {
        "id": "Im7VZEwFIrna"
      },
      "outputs": [],
      "source": [
        "def generate_answer(session_id: str, user_query: str) -> None:\n",
        "    \"\"\"\n",
        "    Generate an answer to the user's query taking chat history into account.\n",
        "\n",
        "    Args:\n",
        "        session_id (str): Session ID to retrieve chat history for.\n",
        "        user_query (str): The user's query string.\n",
        "    \"\"\"\n",
        "    # Initialize list of messages to pass to the chat completion model\n",
        "    messages = []\n",
        "\n",
        "    # Retrieve documents relevant to the user query and convert them to a single string\n",
        "    context = vector_search(user_query)\n",
        "    context = \"\\n\\n\".join([d.get(\"Content\", \"\") for d in context])\n",
        "\n",
        "    # # Create a system prompt containing the retrieved context\n",
        "    # system_message = {\n",
        "    #     \"role\": \"assistant\",\n",
        "    #     \"content\": [\n",
        "    #         {\n",
        "    #             \"text\": f\"Answer the question based only on the following context. If the context is empty, say I DON'T KNOW\\n\\nContext:\\n{context}\",\n",
        "    #         }\n",
        "    #     ]\n",
        "    # }\n",
        "    # # Append the system prompt to the `messages` list\n",
        "    # messages.append(system_message)\n",
        "\n",
        "    # Use the `retrieve_session_history` function to retrieve message history from MongoDB for the session ID `session_id`\n",
        "    # And add all messages in the message history to the `messages` list\n",
        "    message_history = retrieve_session_history(session_id)\n",
        "\n",
        "    messages.extend(message_history)\n",
        "    prompt = create_prompt(user_query)\n",
        "\n",
        "    # Format the user message in the format {\"role\": <role_value>, \"content\": <content_value>}\n",
        "    # The role value for user messages must be \"user\"\n",
        "    # And append the user message to the `messages` list\n",
        "    user_message = {\"role\": \"user\", \"content\": [{ \"text\": prompt}]}\n",
        "\n",
        "\n",
        "    messages.append(user_message)\n",
        "\n",
        "    # Call the chat completions API\n",
        "    client = boto3.client(\n",
        "          service_name='bedrock-runtime',\n",
        "          config=my_config,\n",
        "          aws_access_key_id=os.environ[\"AWS_ACCESS_KEY\"],\n",
        "          aws_secret_access_key=os.environ[\"AWS_SECRET\"],\n",
        "    )\n",
        "\n",
        "    # Set the model ID, e.g., Titan Text Premier.\n",
        "    model_id = \"amazon.titan-text-premier-v1:0\"\n",
        "\n",
        "    # Use the `prompt` created above to populate the `content` field in the chat message\n",
        "    response = client.converse(\n",
        "        modelId=model_id,\n",
        "        messages=messages,\n",
        "        inferenceConfig={\"topP\":0.9},\n",
        "        additionalModelRequestFields={}\n",
        "    )\n",
        "\n",
        "    # Extract the answer from the API response\n",
        "    answer = response[\"output\"][\"message\"][\"content\"][0][\"text\"]\n",
        "\n",
        "    # Use the `store_chat_message` function to store the user message and also the generated answer in the message history collection\n",
        "    # The role value for user messages is \"user\", and \"assistant\" for the generated answer\n",
        "    store_chat_message(session_id, \"user\", user_query)\n",
        "    store_chat_message(session_id, \"assistant\", answer)\n",
        "\n",
        "    print(answer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 84,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZID6RXzIrna",
        "outputId": "73a64f50-459c-4dde-8994-6d799d8c3612"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "I DON'T KNOW\n"
          ]
        }
      ],
      "source": [
        "generate_answer(\n",
        "    session_id=\"1\",\n",
        "    user_query=\"What are triggers in MongoDB Atlas?\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CedtFEZcIrna",
        "outputId": "e5a6246e-8c1f-409c-90ef-1cbceda509c1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "What are triggers in MongoDB Atlas?\n"
          ]
        }
      ],
      "source": [
        "generate_answer(\n",
        "    session_id=\"1\",\n",
        "    user_query=\"What did I just ask you?\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 86,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kny70CA9GFyq",
        "outputId": "cdc868ff-4b46-4efd-ea32-2232ff54634b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Amazon EC2 is a web service that provides resizable compute capacity in the cloud  billed by the hour or second (minimum of 60 seconds)  You can run virtual machines (EC2 instances) ranging in size from one vCPU and one GB memory  to 448 vCPU and 6six TB memory You have a choice of operating systems including Windows Server 2008/2012 /2016/2019  Oracle Linux Red Hat Enterprise Linux and SUSE Linux Elastic Load Balanc ing Elastic Load Balancing automatically distributes incoming application traffic across multiple Amazon EC2 instances For example  you can use Elastic Load Balancing with Amazon EC2 to create a web application that is highly available and scalable\n"
          ]
        }
      ],
      "source": [
        "generate_answer(\n",
        "    session_id=\"1\",\n",
        "    user_query=\"Tell me more about AWS EC2\",\n",
        ")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": ".venv",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.1"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02b591812552444fa6d543779a614097": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8eaed0ab90eb41fa9bdbda63ba6c224e",
            "placeholder": "​",
            "style": "IPY_MODEL_3bc2d9c5f0b240c9a92d780aa0dad2e5",
            "value": " 970/970 [00:00&lt;00:00, 17.9kB/s]"
          }
        },
        "035a50ce208a45258d5b212e8466343f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "08f7a1a01a7043979ad29eac8a0e3d55": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b80749f01934a70bcd8a62def2f1825": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0d7fc927d9994ec28b1a4bd13c818f20": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "147809ed82c7447483165638f21db1bd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19e9a3a1e78243d082e43f9835839f92": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5458a2b4995248e6a39eed0b4a0226aa",
              "IPY_MODEL_449608fbc4804c90bd58f29a281b7619",
              "IPY_MODEL_56b69c3ef27945cfa572afa1ed0741df"
            ],
            "layout": "IPY_MODEL_d6ade1b8e5ac461da386c9321e77eff4"
          }
        },
        "1d8c4f8d764344e8817dee2228cf6dfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20c82bc105d64affbbdd7a43a96441e1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "2300400ebe964c4babb749d995e05e5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "278c4e3d670f4fb4a04117ce3508311f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c8c6eabbd1a4e3c9239dd45ba2fa776",
            "placeholder": "​",
            "style": "IPY_MODEL_6f965b1a2cc349fbaebf1a50f0a40aee",
            "value": "config.json: 100%"
          }
        },
        "280e6088dd084d2cba58a674710c217a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2841d3f13e0a46beb272972e1fa37cec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_08f7a1a01a7043979ad29eac8a0e3d55",
            "max": 8649139,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0b80749f01934a70bcd8a62def2f1825",
            "value": 8649139
          }
        },
        "293e0c5d0f3e407da3e479be176f017e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b74147cb6d5449bb63ed5710509b531": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c2d988b2b8c472dbc0dd75385d7c48f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d36c7f3f08b4dfe95e5bc945f3e1208",
            "max": 141685186,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_de0822612c0f43ebb097ebaf557216f4",
            "value": 141685186
          }
        },
        "2ea0d6344ef24f0e9de5cb7d1951ea2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "37220c85e05c43ccbb114780c027a4e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "375fee84502b4d10885db49b5d5a359f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a0ceaf134d64791bb1e979101658534": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d2131547e4d45e18b6e81fc70e08e91",
            "placeholder": "​",
            "style": "IPY_MODEL_9493053a1712497c98b3489b9fca0c5a",
            "value": " 8.65M/8.65M [00:01&lt;00:00, 8.53MB/s]"
          }
        },
        "3b958bb934ad4a089f135e49e4964a95": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bc2d9c5f0b240c9a92d780aa0dad2e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d38dbf5f34d48d6b0c71e9927b36604": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f00bb98d50f43e39ddb830c3da4a352": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40a21479b8194121960325bf655bb5ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41e1666413ac433688680c53a437180f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "449608fbc4804c90bd58f29a281b7619": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_293e0c5d0f3e407da3e479be176f017e",
            "max": 23,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d51aafbee3e349bb9d28edb80aa83bd6",
            "value": 23
          }
        },
        "46354ca3ee7d4600b78420887a1dde2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f517e78e15d14aa5b37c04d367c99804",
            "placeholder": "​",
            "style": "IPY_MODEL_4e1be448f7494f7eb5b8e09504e37362",
            "value": " 968/968 [00:00&lt;00:00, 39.0kB/s]"
          }
        },
        "4d2131547e4d45e18b6e81fc70e08e91": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e1be448f7494f7eb5b8e09504e37362": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5213f7c794b940a392ecf788cfc34869": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5361e8c2daf24b60bc25e2e2f8df9bf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5458a2b4995248e6a39eed0b4a0226aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4c705996c2c4c2ca2fc0ce6ffb0b210",
            "placeholder": "​",
            "style": "IPY_MODEL_85aacf8449a945e495b6af803c6b694b",
            "value": "added_tokens.json: 100%"
          }
        },
        "56b69c3ef27945cfa572afa1ed0741df": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_147809ed82c7447483165638f21db1bd",
            "placeholder": "​",
            "style": "IPY_MODEL_c72f3aeccd9d4e9d95935890e5e77155",
            "value": " 23.0/23.0 [00:00&lt;00:00, 564B/s]"
          }
        },
        "585f4c87521849288f6ceff6fb7920a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c269701f862549b2b6a91d8ab5838b08",
            "placeholder": "​",
            "style": "IPY_MODEL_1d8c4f8d764344e8817dee2228cf6dfd",
            "value": " 142M/142M [00:05&lt;00:00, 26.6MB/s]"
          }
        },
        "595fb968931940d58025ca940fb51511": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5999315d452b463db818093d62f0c37c",
            "placeholder": "​",
            "style": "IPY_MODEL_5debc9238ea2428e949d3596503367eb",
            "value": "spm.model: 100%"
          }
        },
        "5999315d452b463db818093d62f0c37c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5debc9238ea2428e949d3596503367eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e7ea3dec2d84befbfeca81e59887e37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_595fb968931940d58025ca940fb51511",
              "IPY_MODEL_8a7b2d1688bc49f49fb2dc5feee1aa3b",
              "IPY_MODEL_cbd6067529b44889b050766db92a9c11"
            ],
            "layout": "IPY_MODEL_5f161fc8634148b3bd6c6f576cc5b54c"
          }
        },
        "5f161fc8634148b3bd6c6f576cc5b54c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "611f7ff8cb5d4bffa8b37fb7f33a67e5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65717255d10a4d3a832f916ed177d081": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c8c6eabbd1a4e3c9239dd45ba2fa776": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e8ac2a6989f4ee49f86fca7cca3dab6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_da2e289c84db462194101f8ff02c6e2c",
              "IPY_MODEL_c987709a64204d99bc959a44ad27332e",
              "IPY_MODEL_02b591812552444fa6d543779a614097"
            ],
            "layout": "IPY_MODEL_0d7fc927d9994ec28b1a4bd13c818f20"
          }
        },
        "6f965b1a2cc349fbaebf1a50f0a40aee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7758f0d9087749eda61f44458096985d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20c82bc105d64affbbdd7a43a96441e1",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f3fcd4b0f25f46f88e37abea32725ac1",
            "value": 0
          }
        },
        "7a11f8bed3e54e718f12a778ce78088b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3d38dbf5f34d48d6b0c71e9927b36604",
            "placeholder": "​",
            "style": "IPY_MODEL_375fee84502b4d10885db49b5d5a359f",
            "value": "tokenizer.json: 100%"
          }
        },
        "7a6139f6ae86449aae7a95aec0d5d3f6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c7df8af99004089800e6892af658576": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_611f7ff8cb5d4bffa8b37fb7f33a67e5",
            "max": 1447,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9fadd4111cb24606942c68816e2450f5",
            "value": 1447
          }
        },
        "85aacf8449a945e495b6af803c6b694b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85e3db77cb7e46718689089639713b10": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d5b227dedc4346998f88260b04d0600c",
              "IPY_MODEL_2c2d988b2b8c472dbc0dd75385d7c48f",
              "IPY_MODEL_585f4c87521849288f6ceff6fb7920a8"
            ],
            "layout": "IPY_MODEL_41e1666413ac433688680c53a437180f"
          }
        },
        "8a7b2d1688bc49f49fb2dc5feee1aa3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a221234962054cb2b29ff66abe258d5c",
            "max": 2464616,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_40a21479b8194121960325bf655bb5ce",
            "value": 2464616
          }
        },
        "8c62eeeed4844094873a379ec87a778c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a11f8bed3e54e718f12a778ce78088b",
              "IPY_MODEL_2841d3f13e0a46beb272972e1fa37cec",
              "IPY_MODEL_3a0ceaf134d64791bb1e979101658534"
            ],
            "layout": "IPY_MODEL_b8ffdb1a268a4c33aa07ff14c6d32e49"
          }
        },
        "8eaed0ab90eb41fa9bdbda63ba6c224e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9493053a1712497c98b3489b9fca0c5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9c435b51c3624a3e89ec9712ddb6e523": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d36c7f3f08b4dfe95e5bc945f3e1208": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9fadd4111cb24606942c68816e2450f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a221234962054cb2b29ff66abe258d5c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8e895a8ac6c4b31ac27401e8b59969b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa3f5fe0a6df4288a833a6dc9c7d839e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_278c4e3d670f4fb4a04117ce3508311f",
              "IPY_MODEL_ccaee1a3af4743f9a441170180db3a5f",
              "IPY_MODEL_46354ca3ee7d4600b78420887a1dde2c"
            ],
            "layout": "IPY_MODEL_b25684ada2ca4c3ba38623e75ffa6e27"
          }
        },
        "b25684ada2ca4c3ba38623e75ffa6e27": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4c705996c2c4c2ca2fc0ce6ffb0b210": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b8ffdb1a268a4c33aa07ff14c6d32e49": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c269701f862549b2b6a91d8ab5838b08": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2f8658ef30849c8b70e362b79766309": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c71afb2162b94738bbf93b1976097a08": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c72f3aeccd9d4e9d95935890e5e77155": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c987709a64204d99bc959a44ad27332e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b74147cb6d5449bb63ed5710509b531",
            "max": 970,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_37220c85e05c43ccbb114780c027a4e7",
            "value": 970
          }
        },
        "cbd6067529b44889b050766db92a9c11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc23f068cf1e489fb9254a128290250e",
            "placeholder": "​",
            "style": "IPY_MODEL_5361e8c2daf24b60bc25e2e2f8df9bf2",
            "value": " 2.46M/2.46M [00:00&lt;00:00, 12.2MB/s]"
          }
        },
        "cc23f068cf1e489fb9254a128290250e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc7ace7ad1644083bd7362cb03795dea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cce96eb1a33a4480b933e5211397be9e",
            "placeholder": "​",
            "style": "IPY_MODEL_2ea0d6344ef24f0e9de5cb7d1951ea2d",
            "value": ""
          }
        },
        "ccaee1a3af4743f9a441170180db3a5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2f8658ef30849c8b70e362b79766309",
            "max": 968,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2300400ebe964c4babb749d995e05e5a",
            "value": 968
          }
        },
        "cce96eb1a33a4480b933e5211397be9e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d429e62af04a4becbb784ad6041221a4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d51aafbee3e349bb9d28edb80aa83bd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d5a4f9f8c31842b2af82a15670d9dc6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5b227dedc4346998f88260b04d0600c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c71afb2162b94738bbf93b1976097a08",
            "placeholder": "​",
            "style": "IPY_MODEL_035a50ce208a45258d5b212e8466343f",
            "value": "model.safetensors: 100%"
          }
        },
        "d6ade1b8e5ac461da386c9321e77eff4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da2e289c84db462194101f8ff02c6e2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f00bb98d50f43e39ddb830c3da4a352",
            "placeholder": "​",
            "style": "IPY_MODEL_280e6088dd084d2cba58a674710c217a",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "de0822612c0f43ebb097ebaf557216f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "deb411770ca14acea2132206fcb54bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cc7ace7ad1644083bd7362cb03795dea",
              "IPY_MODEL_7758f0d9087749eda61f44458096985d",
              "IPY_MODEL_fb20472b5c944892933653f4fd1e0694"
            ],
            "layout": "IPY_MODEL_65717255d10a4d3a832f916ed177d081"
          }
        },
        "e115fd45185441888a6e45d00f9e0260": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b958bb934ad4a089f135e49e4964a95",
            "placeholder": "​",
            "style": "IPY_MODEL_5213f7c794b940a392ecf788cfc34869",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "eb00279c0176456db4a44802acefbaa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e115fd45185441888a6e45d00f9e0260",
              "IPY_MODEL_7c7df8af99004089800e6892af658576",
              "IPY_MODEL_fa72acef332e48b5948c69cf2e9b2573"
            ],
            "layout": "IPY_MODEL_d429e62af04a4becbb784ad6041221a4"
          }
        },
        "f3fcd4b0f25f46f88e37abea32725ac1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f517e78e15d14aa5b37c04d367c99804": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa72acef332e48b5948c69cf2e9b2573": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8e895a8ac6c4b31ac27401e8b59969b",
            "placeholder": "​",
            "style": "IPY_MODEL_d5a4f9f8c31842b2af82a15670d9dc6e",
            "value": " 1.45k/1.45k [00:00&lt;00:00, 64.4kB/s]"
          }
        },
        "fb20472b5c944892933653f4fd1e0694": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a6139f6ae86449aae7a95aec0d5d3f6",
            "placeholder": "​",
            "style": "IPY_MODEL_9c435b51c3624a3e89ec9712ddb6e523",
            "value": " 0/0 [00:00&lt;?, ?it/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}