Merge pull request #4078 from aboutcode-org/fix-package-scan-bugs

Fix conda environment yaml parsing errors

Author: AyanSinhaMahapatra

src/packagedcode/conda.py

@@ -378,6 +378,7 @@ def parse(cls, location, package_only=False):
                 primary_language=cls.default_primary_language,
                 dependencies=dependencies,
                 extra_data=extra_data,
+                is_private=True,
             )
             yield models.PackageData.from_data(package_data, package_only)
 
@@ -536,6 +537,8 @@ def get_conda_yaml_dependencies(conda_data):
 
             if "::" in dep:
                 namespace, dep = dep.split("::")
+                if "/" in namespace or ":" in namespace:
+                    namespace = None
 
             req = parse_requirement_line(dep)
             if req:

tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json

@@ -33,7 +33,7 @@
     "notice_text": null,
     "source_packages": [],
     "file_references": [],
-    "is_private": false,
+    "is_private": true,
     "is_virtual": false,
     "extra_data": {
       "channels": [

tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json

@@ -33,7 +33,7 @@
     "notice_text": null,
     "source_packages": [],
     "file_references": [],
-    "is_private": false,
+    "is_private": true,
     "is_virtual": false,
     "extra_data": {
       "channels": [

tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml

@@ -0,0 +1,3 @@
+name: test
+dependencies:
+  - https://104.17.92.24:443::flask=1.0.2

tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml-expected.json

@@ -0,0 +1,58 @@
+[
+  {
+    "type": "conda",
+    "namespace": null,
+    "name": "test",
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "Python",
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": true,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [
+      {
+        "purl": "pkg:conda/[email protected]",
+        "extracted_requirement": "=1.0.2",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": true,
+        "resolved_package": {},
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "conda_yaml",
+    "purl": "pkg:conda/test"
+  }
+]

tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json

@@ -33,7 +33,7 @@
     "notice_text": null,
     "source_packages": [],
     "file_references": [],
-    "is_private": false,
+    "is_private": true,
     "is_virtual": false,
     "extra_data": {
       "channels": [

tests/packagedcode/test_conda.py

@@ -115,6 +115,12 @@ def test_parse_conda_yaml_simple_dependencies(self):
         expected_loc = self.get_test_loc('conda/conda-yaml/ringer/environment.yaml-expected.json')
         self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
 
+    def test_parse_conda_yaml_does_not_fail_on_test_files_with_port(self):
+        test_file = self.get_test_loc('conda/conda-yaml/test/environment_host_port.yml')
+        package = conda.CondaYamlHandler.parse(test_file)
+        expected_loc = self.get_test_loc('conda/conda-yaml/test/environment_host_port.yml-expected.json')
+        self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+
     def test_conda_get_conda_meta_json(self):
         meta_yaml_path = 'conda/pkgs/requests-2.32.3-py312h06a4308_1/info/recipe/meta.yaml'
         conda_meta_json_path = 'conda/conda-meta/requests-2.32.3-py312h06a4308_1.json'