scrapinghub · renancunha · Aug 18, 2020
diff --git a/tests/resources/bv_amazon_product.json b/tests/resources/bv_amazon_product.json
@@ -0,0 +1,113 @@
+{
+  "is_faux_price": null,
+  "shipping_available": null,
+  "features": [
+    [
+      "amazon bestsellers rank",
+      "1,746 in Computers & Accessories (See Top 100 in Computers & Accessories) #186 in Ink Cartridges"
+    ],
+    [
+      "asin",
+      "B07VWYM7KX"
+    ],
+    [
+      "brand name",
+      "Epson"
+    ],
+    [
+      "colour",
+      "Black"
+    ],
+    [
+      "customer reviews",
+      "73 customer ratings"
+    ],
+    [
+      "date first available",
+      "2 Aug. 2019"
+    ],
+    [
+      "item weight",
+      "59 g"
+    ],
+    [
+      "manufacturer part number",
+      "EP66635"
+    ],
+    [
+      "model number",
+      "603XL"
+    ],
+    [
+      "number of items",
+      "1"
+    ],
+    [
+      "package dimensions",
+      "14 x 9.5 x 2.3 cm"
+    ],
+    [
+      "price",
+      "\u00a331.19"
+    ],
+    [
+      "shipping weight",
+      "59 g"
+    ],
+    [
+      "size",
+      "One Size"
+    ]
+  ],
+  "weight": "59 g",
+  "sold_by": null,
+  "brand_name": "Epson",
+  "is_buybox_winner": null,
+  "download_screenshot": null,
+  "currency": "£",
+  "video_available": null,
+  "download_html": null,
+  "country_code": null,
+  "images": null,
+  "members_only_item": null,
+  "category": "Printers & Accessories > Printer Accessories > Ink Cartridges",
+  "dimensions": "14 x 9.5 x 2.3 cm",
+  "part_number": "603XL",
+  "get_video": null,
+  "datapoint_identifier": null,
+  "zipcode": null,
+  "get_ratings_extended_limit": null,
+  "identifier_sku": "B07VWYM7KX",
+  "description_bulletpoints": null,
+  "stock_inventory": null,
+  "description": "This Epson Starfish 603XL Black Ink Cartridge is for use in Epson Expression Home XP-2100, XP-2105, XP-3100, XP3105, XP-4100, XP-4105, WorkForce WF-2810DWF, WF-2830DWF, WF-2835DWF and WF-2850DWF inkjet printers. The cartridge contains 8.9ml of ink, with a high print yield of up to 500 pages. This pack contains 1 black ink cartridge.",
+  "response_url": "https://www.amazon.co.uk/dp/B07VWYM7KX",
+  "discovery_item_id": null,
+  "image_urls": [
+    "https://images-na.ssl-images-amazon.com/images/I/6177Iwq1r8L._AC_SL1000_.jpg"
+  ],
+  "available_online": null,
+  "in_stock": true,
+  "rating_current": "4.4",
+  "fulfilled_by": null,
+  "ranking_item": null,
+  "get_shipping_info": null,
+  "preorder": null,
+  "get_ratings": null,
+  "name": "Epson Original Ink Cartridge Compatible with Expression Home/Workforce Series, 500 Pages, Black, One Size, 603XL",
+  "url": "https://www.amazon.co.uk/dp/B07VWYM7KX",
+  "country": null,
+  "colour": "Black",
+  "price_now": "31.19",
+  "is_add_on_item": null,
+  "reviews_quantity_1_star": "4",
+  "reviews_quantity_2_star": "2",
+  "reviews_quantity_3_star": "5",
+  "reviews_quantity_4_star": "12",
+  "reviews_quantity_5_star": "49",
+  "rating_out_of": "5",
+  "download_product_image": null,
+  "get_ratings_extended": null,
+  "amazons_choice": null,
+  "reviews_quantity": "72"
+}
diff --git a/tests/resources/unified_schema_product.json b/tests/resources/unified_schema_product.json
@@ -0,0 +1,121 @@
+{
+  "additionalProperty": [
+    {
+      "name": "amazon bestsellers rank",
+      "value": "1,746 in Computers & Accessories (See Top 100 in Computers & Accessories) #186 in Ink Cartridges"
+    },
+    {
+      "name": "asin",
+      "value": "B07VWYM7KX"
+    },
+    {
+      "name": "brand name",
+      "value": "Epson"
+    },
+    {
+      "name": "colour",
+      "value": "Black"
+    },
+    {
+      "name": "customer reviews",
+      "value": "73 customer ratings"
+    },
+    {
+      "name": "date first available",
+      "value": "2 Aug. 2019"
+    },
+    {
+      "name": "item weight",
+      "value": "59 g"
+    },
+    {
+      "name": "manufacturer part number",
+      "value": "EP66635"
+    },
+    {
+      "name": "model number",
+      "value": "603XL"
+    },
+    {
+      "name": "number of items",
+      "value": "1"
+    },
+    {
+      "name": "package dimensions",
+      "value": "14 x 9.5 x 2.3 cm"
+    },
+    {
+      "name": "price",
+      "value": "£31.19"
+    },
+    {
+      "name": "shipping weight",
+      "value": "59 g"
+    },
+    {
+      "name": "size",
+      "value": "One Size"
+    }
+  ],
+  "aggregateRating": {
+    "bestRating": 5,
+    "ratingValue": 4.4,
+    "reviewCount": 72
+  },
+  "ratingHistogram": [
+    {
+      "ratingValue": "5",
+      "ratingPercentage": 49
+    },
+    {
+      "ratingValue": "4",
+      "ratingPercentage": 12
+    },
+    {
+      "ratingValue": "3",
+      "ratingPercentage": 5
+    },
+    {
+      "ratingValue": "2",
+      "ratingPercentage": 2
+    },
+    {
+      "ratingValue": "1",
+      "ratingPercentage": 4
+    }
+  ],
+  "brand": "Epson",
+  "breadcrumbs": [
+    {
+      "link": "https://www.amazon.co.uk/b/ref=dp_bc_aui_C_1/261-4427288-0767136?ie=UTF8&node=17477983031",
+      "name": "Printers & Accessories"
+    },
+    {
+      "link": "https://www.amazon.co.uk/b/ref=dp_bc_aui_C_2/261-4427288-0767136?ie=UTF8&node=430440031",
+      "name": "Printer Accessories"
+    },
+    {
+      "link": "https://www.amazon.co.uk/Ink-Cartridges/b/ref=dp_bc_aui_C_3/261-4427288-0767136?ie=UTF8&node=430492031",
+      "name": "Ink Cartridges"
+    }
+  ],
+  "description": "This Epson Starfish 603XL Black Ink Cartridge is for use in Epson Expression Home XP-2100, XP-2105, XP-3100, XP3105, XP-4100, XP-4105, WorkForce WF-2810DWF, WF-2830DWF, WF-2835DWF and WF-2850DWF inkjet printers. The cartridge contains 8.9ml of ink, with a high print yield of up to 500 pages. This pack contains 1 black ink cartridge.",
+  "gtin": null,
+  "images": [
+    "https://images-na.ssl-images-amazon.com/images/I/6177Iwq1r8L._AC_SL1000_.jpg"
+  ],
+  "mainImage": "https://images-na.ssl-images-amazon.com/images/I/6177Iwq1r8L._AC_SL1000_.jpg",
+  "mpn": "EP66635",
+  "name": "Epson Original Ink Cartridge Compatible with Expression Home/Workforce Series, 500 Pages, Black, One Size, 603XL",
+  "offers": [
+    {
+      "availability": "InStock",
+      "currency": "£",
+      "price": "31.19",
+      "regularPrice": "31.19"
+    }
+  ],
+  "probability": 0.9877380132675171,
+  "sku": null,
+  "url": "https://www.amazon.co.uk/dp/B07VWYM7KX"
+}
diff --git a/tests/test_regression.py b/tests/test_regression.py
@@ -2,6 +2,7 @@
 """
 
 from shublang import evaluate
+from .utils import traverse_specs, get_resource_data
 
 
 def test_get_non_empty_price():
@@ -12,3 +13,70 @@ def test_get_non_empty_price():
     assert data == ['price: 123823.00']
 
     assert evaluate('re_search("(\d+\.\d{2})") | first | float | first', data) == 123823.00
+
+
+def test_convert_product_to_bv_product_schema():
+    """Converting Unified-Schema product data to a custom customer schema (BV - Amazon).
+    """
+
+    specifications = {
+        "name": 'jmespath("name") | first',
+        "brand_name": 'jmespath("brand") | first',
+        "description": 'jmespath("description") | first',
+        "category": 'jmespath("breadcrumbs[*].name") | first | join(" > ")',
+        "identifier_sku": "jmespath(\"additionalProperty[?name=='asin'].value | [0]\") | first",
+        "part_number": "jmespath(\"additionalProperty[?name=='model number'].value | [0]\") | first",
+        "features": 'jmespath("additionalProperty[].[name, value]") | first',
+        "weight": "jmespath(\"additionalProperty[?name=='item weight'].value | [0]\") | first",
+        "dimensions": "jmespath(\"additionalProperty[?name=='package dimensions'].value | [0]\") | first",
+        "colour": "jmespath(\"additionalProperty[?name=='colour'].value | [0]\") | first",
+        "url": 'jmespath("url") | first',
+        "response_url": 'jmespath("url") | first',
+        "image_urls": 'jmespath("images") | first',
+        "price_now": 'jmespath("offers[*].price | [0]") | first',
+        "currency": 'jmespath("offers[*].currency | [0]") | first',
+        "in_stock": 'jmespath("offers[*].availability | [0]") | map(lambda x: x == "InStock") | first',
+        "rating_current": 'jmespath("aggregateRating.ratingValue") | map(lambda x: str(x)) | first',
+        "rating_out_of": 'jmespath("aggregateRating.bestRating") | map(lambda x: str(x)) | first',
+        "reviews_quantity": 'jmespath("aggregateRating.reviewCount") | map(lambda x: str(x)) | first',
+        "reviews_quantity_1_star": "jmespath(\"ratingHistogram[?ratingValue=='1'].ratingPercentage | [0]\") | map(lambda x: str(x)) | first",
+        "reviews_quantity_2_star": "jmespath(\"ratingHistogram[?ratingValue=='2'].ratingPercentage | [0]\") | map(lambda x: str(x)) | first",
+        "reviews_quantity_3_star": "jmespath(\"ratingHistogram[?ratingValue=='3'].ratingPercentage | [0]\") | map(lambda x: str(x)) | first",
+        "reviews_quantity_4_star": "jmespath(\"ratingHistogram[?ratingValue=='4'].ratingPercentage | [0]\") | map(lambda x: str(x)) | first",
+        "reviews_quantity_5_star": "jmespath(\"ratingHistogram[?ratingValue=='5'].ratingPercentage | [0]\") | map(lambda x: str(x)) | first",
+
+        # missing fields
+        "is_add_on_item": None,
+        "download_product_image": None,
+        "amazons_choice": None,
+        "get_ratings_extended": None,
+        "country": None,
+        "fulfilled_by": None,
+        "ranking_item": None,
+        "get_shipping_info": None,
+        "preorder": None,
+        "get_ratings": None,
+        "available_online": None,
+        "discovery_item_id": None,
+        "stock_inventory": None,
+        "description_bulletpoints": None,
+        "get_video": None,
+        "datapoint_identifier": None,
+        "zipcode": None,
+        "get_ratings_extended_limit": None,
+        "video_available": None,
+        "download_html": None,
+        "country_code": None,
+        "images": None,
+        "members_only_item": None,
+        "is_buybox_winner": None,
+        "download_screenshot": None,
+        "sold_by": None,
+        "is_faux_price": None,
+        "shipping_available": None,
+    }
+
+    product = get_resource_data("unified_schema_product.json")
+    output = traverse_specs(specifications.copy(), product)
+
+    assert output == get_resource_data("bv_amazon_product.json")
diff --git a/tests/utils.py b/tests/utils.py
@@ -0,0 +1,23 @@
+import json
+from shublang.shublang import evaluate
+
+
+def traverse_specs(specs, data):
+    """Given a specs object, it will evaluate all the shublang expressions found,
+    populating it with `data` and returning the newly filled object.
+    """
+    for key in specs:
+        if isinstance(key, str):
+            if specs[key]:
+                specs[key] = evaluate(specs[key], [data])
+        else:
+            specs[key] = traverse_specs(specs[key], data)
+
+    return specs
+
+
+def get_resource_data(name):
+    with open(f"tests/resources/{name}", "r") as f:
+        data = json.loads(f.read())
+
+    return data