Merge pull request #2932 from nexB/fix-2911-purl-less-deps

Fix package/dependency creation bugs

Author: pombredanne

.gitignore

@@ -10,6 +10,7 @@
 # virtualenv and other misc bits
 /src/*.egg-info
 *.egg-info
+!tests/packagedcode/data/pypi/source-package/pip-22.0.4/src/pip.egg-info
 /dist
 /build
 /bin

src/packagedcode/alpine.py

@@ -129,7 +129,7 @@ def compute_normalized_license(cls, package):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        cls.assign_package_to_parent_tree(
+        models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,

src/packagedcode/build.py

@@ -51,7 +51,7 @@ def parse(cls, location):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        cls.assign_package_to_parent_tree(
+        models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
@@ -376,7 +376,7 @@ def compute_normalized_license(cls, package):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        cls.assign_package_to_parent_tree(
+        models.DatafileHandler.assign_package_to_parent_tree(
             package_=package,
             resource=resource,
             codebase=codebase,

src/packagedcode/build_gradle.py

@@ -35,7 +35,7 @@ def parse(cls, location):
     # TODO: handle complex cases of nested builds with many packages
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        cls.assign_package_to_parent_tree(
+        models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,

src/packagedcode/conda.py

@@ -71,7 +71,7 @@ def get_conda_root(cls, resource, codebase):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        return super().assign_package_to_resources(
+        return models.DatafileHandler.assign_package_to_resources(
             package=package,
             resource=cls.get_conda_root(resource, codebase),
             codebase=codebase,

src/packagedcode/debian.py

@@ -67,7 +67,7 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         # only assign this resource
-        return super().assign_package_to_resources(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_resources(package, resource, codebase)
 
 
 # TODO: introspect archive
@@ -92,7 +92,7 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         # only assign this resource
-        return super().assign_package_to_resources(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_resources(package, resource, codebase)
 
 
 # TODO: introspect archive
@@ -117,7 +117,7 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         # only assign this resource
-        return super().assign_package_to_resources(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_resources(package, resource, codebase)
 
 
 # TODO: also look into neighboring md5sum and data.tarball copyright files!!!
@@ -142,7 +142,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         # two levels up
         root = resource.parent(codebase).parent(codebase)
         if root:
-            return cls.assign_package_to_resources(package, root, codebase)
+            return models.DatafileHandler.assign_package_to_resources(package, root, codebase)
 
 
 # TODO: also look into neighboring copyright files!!!
@@ -169,7 +169,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         # two levels up
         root = resource.parent(codebase).parent(codebase)
         if root:
-            return cls.assign_package_to_resources(package, root, codebase)
+            return models.DatafileHandler.assign_package_to_resources(package, root, codebase)
 
 
 class DebianDscFileHandler(models.DatafileHandler):
@@ -198,7 +198,7 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         # only assign this resource
-        return super().assign_package_to_resources(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_resources(package, resource, codebase)
 
 
 class DebianInstalledStatusDatabaseHandler(models.DatafileHandler):
@@ -489,7 +489,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         # two levels up
         root = resource.parent(codebase).parent(codebase)
         if root:
-            return cls.assign_package_to_resources(package, root, codebase)
+            return models.DatafileHandler.assign_package_to_resources(package, root, codebase)
 
 
 def build_package_data_from_package_filename(filename, datasource_id, package_type,):

src/packagedcode/godeps.py

@@ -75,7 +75,7 @@ def parse(cls, location):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        super().assign_package_to_parent_tree(package, resource, codebase)
+        models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
 @attr.s

src/packagedcode/jar_manifest.py

@@ -53,7 +53,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
             parent = resource.parent(codebase)
 
         if parent:
-            super().assign_package_to_resources(package, resource=parent, codebase=codebase)
+            models.DatafileHandler.assign_package_to_resources(package, resource=parent, codebase=codebase)
 
 
 

src/packagedcode/maven.py

@@ -115,7 +115,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
 
         if resource.path.endswith('.pom'):
             # we only treat the parent as the root
-            return super().assign_package_to_parent_tree(package, resource, codebase)
+            return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
         # the root is either the parent or further up for poms stored under
         # a META-INF dir
@@ -139,7 +139,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         if not root:
             root = resource.parent(codebase)
 
-        return super().assign_package_to_resources(package, resource=root, codebase=codebase)
+        return models.DatafileHandler.assign_package_to_resources(package, resource=root, codebase=codebase)
 
     @classmethod
     def compute_normalized_license(cls, package):

src/packagedcode/models.py

@@ -438,12 +438,17 @@ def from_dependent_packages(
         """
         dependent_packages = dependent_packages or []
         for dependent_package in dependent_packages:
-            yield Dependency.from_dependent_package(
-                dependent_package=dependent_package,
-                datafile_path=datafile_path,
-                datasource_id=datasource_id,
-                package_uid=package_uid,
-            )
+            if dependent_package.purl:
+                yield Dependency.from_dependent_package(
+                    dependent_package=dependent_package,
+                    datafile_path=datafile_path,
+                    datasource_id=datasource_id,
+                    package_uid=package_uid,
+                )
+            else:
+                if TRACE:
+                    logger_debug(f' Dependency.from_dependent_packages: dependent_package (does not have purl): {dependent_package}')
+                pass
 
 
 @attr.attributes(slots=True)

src/packagedcode/npm.py

@@ -53,11 +53,18 @@ def assemble(cls, package_data, resource, codebase):
         else:
             dir_resource=resource
 
-        yield from cls.assemble_from_many_datafiles(
+        for assembled in cls.assemble_from_many_datafiles(
             datafile_name_patterns=datafile_name_patterns,
             directory=dir_resource,
             codebase=codebase,
-        )
+        ):
+            if isinstance(assembled, models.Package):
+                cls.assign_package_to_resources(
+                    package=assembled,
+                    resource=resource,
+                    codebase=codebase,
+                )
+            yield assembled
 
     @classmethod
     def walk_npm(cls, resource, codebase, depth=0):

src/packagedcode/opam.py

@@ -119,7 +119,7 @@ def parse(cls, location):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        return super().assign_package_to_parent_tree(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
 def get_repository_homepage_url(name):

src/packagedcode/phpcomposer.py

@@ -45,7 +45,7 @@ def assemble(cls, package_data, resource, codebase):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        return super().assign_package_to_parent_tree(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
     @classmethod
     def compute_normalized_license(cls, package):

src/packagedcode/plugin_package.py

@@ -158,47 +158,53 @@ def create_package_and_deps(codebase, **kwargs):
         for package_data in resource.package_data:
             try:
                 package_data = PackageData.from_dict(package_data)
-            except Exception as e:
-                raise Exception(
-                    'create_package_and_deps: Failed to create PackageData:',
-                    package_data,
-                ) from e
-
-            if TRACE:
-                logger_debug('  create_package_and_deps: package_data:', package_data)
-
-            # Find a handler for this package datasource to assemble collect
-            # packages and deps
-            handler = get_package_handler(package_data)
-            if TRACE:
-                logger_debug('  create_package_and_deps: handler:', handler)
-            click.echo(f'  create_package_and_deps: handler: {handler}')
-            items = handler.assemble(
-                package_data=package_data,
-                resource=resource,
-                codebase=codebase,
-            )
-
-            for item in items:
+
                 if TRACE:
-                    logger_debug('    create_package_and_deps: item:', item)
+                    logger_debug('  create_package_and_deps: package_data:', package_data)
 
-                if isinstance(item, Package):
-                    packages_top_level.append(item)
+                # Find a handler for this package datasource to assemble collect
+                # packages and deps
+                handler = get_package_handler(package_data)
+                if TRACE:
+                    logger_debug('  create_package_and_deps: handler:', handler)
+                click.echo(f'  create_package_and_deps: handler: {handler}')
+ 
+                items = handler.assemble(
+                    package_data=package_data,
+                    resource=resource,
+                    codebase=codebase,
+                )
+
+                for item in items:
+                    if TRACE:
+                        logger_debug('    create_package_and_deps: item:', item)
 
-                elif isinstance(item, Dependency):
-                    dependencies_top_level.append(item)
+                    if isinstance(item, Package):
+                        packages_top_level.append(item)
 
-                elif isinstance(item, Resource):
-                    seen_resource_ids.add(item.rid)
-                    if TRACE:
-                        logger_debug(
-                            '    create_package_and_deps: seen_resource_ids:',
-                            seen_resource_ids,
-                        )
+                    elif isinstance(item, Dependency):
+                        dependencies_top_level.append(item)
 
-                else:
-                    raise Exception(f'Unknown package assembly item type: {item!r}')
+                    elif isinstance(item, Resource):
+                        seen_resource_ids.add(item.rid)
+                        if TRACE:
+                            logger_debug(
+                                '    create_package_and_deps: seen_resource_ids:',
+                                seen_resource_ids,
+                            )
+
+                    else:
+                        raise Exception(f'Unknown package assembly item type: {item!r}')
+
+            except Exception as e:
+                msg = f'create_package_and_deps: Failed to assemble PackageData: {package_data}: \n {e}'
+                resource.scan_errors.append(msg)
+                resource.save(codebase)
+
+                if TRACE:
+                    import traceback
+                    msg += traceback.format_exc()
+                    raise Exception(msg) from e
 
     codebase.attributes.packages.extend(pkg.to_dict() for pkg in packages_top_level)
     codebase.attributes.dependencies.extend(dep.to_dict() for dep in dependencies_top_level)

src/packagedcode/pypi.py

@@ -93,7 +93,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
         # two levels up
         root = resource.parent(codebase).parent(codebase)
         if root:
-            return cls.assign_package_to_resources(package, root, codebase)
+            return models.DatafileHandler.assign_package_to_resources(package, root, codebase)
 
 
 class PythonEditableInstallationPkgInfoFile(BasePypiHandler):
@@ -115,7 +115,7 @@ def parse(cls, location):
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
         # only the parent for now... though it can be more complex
-        return cls.assign_package_to_parent_tree(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
 class BaseExtractedPythonLayout(BasePypiHandler):
@@ -144,7 +144,7 @@ def assemble(cls, package_data, resource, codebase):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        return cls.assign_package_to_parent_tree(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
 class PythonSdistPkgInfoFile(BaseExtractedPythonLayout):

src/packagedcode/rubygems.py

@@ -123,7 +123,7 @@ def assemble(cls, package_data, resource, codebase):
 
     @classmethod
     def assign_package_to_resources(cls, package, resource, codebase):
-        return cls.assign_package_to_parent_tree(package, resource, codebase)
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
 class GemspecHandler(BaseGemHandler):