Merge pull request #548 from aboutcode-org/matchcode-update

Add fragment matching to MatchCode

Author: JonoYang

Makefile

@@ -16,11 +16,18 @@ ACTIVATE?=. ${VENV}/bin/activate;
 VIRTUALENV_PYZ=../etc/thirdparty/virtualenv.pyz
 # Do not depend on Python to generate the SECRET_KEY
 GET_SECRET_KEY=`base64 /dev/urandom | head -c50`
+
 # Customize with `$ make envfile ENV_FILE=/etc/purldb/.env`
 ENV_FILE=.env
+
 # Customize with `$ make postgres PACKAGEDB_DB_PASSWORD=YOUR_PASSWORD`
 PACKAGEDB_DB_PASSWORD=packagedb
 MATCHCODEIO_DB_PASSWORD=matchcodeio
+SCANCODEIO_DB_PASSWORD=scancodeio
+
+# Django settings shortcuts
+DJSM_PDB=DJANGO_SETTINGS_MODULE=purldb_project.settings
+DJSM_MAT=DJANGO_SETTINGS_MODULE=matchcode_project.settings
 
 # Use sudo for postgres, but only on Linux
 UNAME := $(shell uname)
@@ -56,22 +63,17 @@ envfile_testing: envfile
 	@echo SCANCODEIO_DB_USER=\"postgres\" >> ${ENV_FILE}
 	@echo SCANCODEIO_DB_PASSWORD=\"postgres\" >> ${ENV_FILE}
 
-doc8:
-	@echo "-> Run doc8 validation"
-	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/
-
 valid:
 	@echo "-> Run Ruff format"
 	@${ACTIVATE} ruff format  --exclude etc/scripts/ --exclude purldb-toolkit/ --exclude purl2vcs/
 	@echo "-> Run Ruff linter"
 	@${ACTIVATE} ruff check --fix --exclude etc/scripts/ --exclude purldb-toolkit/ --exclude purl2vcs/
 
-check:
+check: check_docs
 	@echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)"
 	@${ACTIVATE} ruff check --exclude etc/scripts/ --exclude purldb-toolkit/ --exclude purl2vcs/
 	@echo "-> Run Ruff format validation"
 	@${ACTIVATE} ruff format --check --exclude etc/scripts/ --exclude purldb-toolkit/ --exclude purl2vcs/
-	@$(MAKE) doc8
 
 clean:
 	@echo "-> Clean the Python env"
@@ -102,7 +104,7 @@ postgres_matchcodeio:
 	@echo "-> Create 'matchcodeio' database"
 	${SUDO_POSTGRES} createdb --encoding=utf-8 --owner=matchcodeio matchcodeio
 	${MATCHCODE_MANAGE} migrate
-
+	
 run:
 	${MANAGE} runserver 8001 --insecure
 
@@ -118,13 +120,20 @@ run_visit: seed
 run_map:
 	${MANAGE} run_map
 
-test:
-	@echo "-> Run the test suite"
-	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs --ignore matchcode_pipeline --ignore matchcode_project --ignore purldb-toolkit --ignore packagedb/tests/test_throttling.py
-	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs packagedb/tests/test_throttling.py
-	${ACTIVATE} DJANGO_SETTINGS_MODULE=matchcode_project.settings ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline
-	${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs purldb-toolkit/
-	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs purl2vcs
+test_purldb:
+	${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf minecode packagedb purl2vcs purldb_project purldb_public_project --ignore packagedb/tests/test_throttling.py 
+	${ACTIVATE} ${DJSM_PDB} pytest -vvs --lf packagedb/tests/test_throttling.py
+
+test_toolkit:
+	${ACTIVATE} pytest -vvs purldb-toolkit/
+
+test_clearcode:
+	${ACTIVATE} ${DJSM_PDB} ${PYTHON_EXE} -m pytest -vvs clearcode clearindex
+
+test_matchcode:
+	${ACTIVATE} ${DJSM_MAT} ${PYTHON_EXE} -m pytest -vvs matchcode_pipeline matchcode-toolkit matchcode
+
+test: test_purldb test_matchcode test_toolkit test_clearcode
 
 shell:
 	${MANAGE} shell
@@ -153,7 +162,7 @@ check_docs:
 	@echo "Check Sphinx Documentation build minimally"
 	@${ACTIVATE} sphinx-build -E -W docs/source build
 	@echo "Check for documentation style errors"
-	@${ACTIVATE} doc8 --max-line-length 100 docs/source --ignore D000 --quiet
+	@${ACTIVATE} doc8 --max-line-length 100 docs/source --ignore-path docs/_build/ --ignore D000 --quiet
 
 docker-images:
 	@echo "-> Build Docker services"
@@ -164,4 +173,5 @@ docker-images:
 	@mkdir -p dist/
 	@docker save minecode minecode_minecode nginx | gzip > dist/minecode-images-`git describe --tags`.tar.gz
 
-.PHONY: virtualenv conf dev envfile isort black doc8 valid check clean migrate postgres run test shell clearsync clearindex index_packages bump docs docker-images
+# keep this sorted
+.PHONY: black bump check check_docs clean `clearindex clearsync conf dev docker-images docs envfile envfile_testing index_packages isort migrate postgres postgres_matchcodeio priority_queue run run_map run_matchcodeio run_visit seed shell test test_clearcode test_matchcode test_purldb test_toolkit valid virtualenv

README.rst

@@ -1,24 +1,23 @@
+==========
 The purldb
 ==========
-This repo consists of these main tools:
 
-- PackageDB that is the reference model (based on ScanCode toolkit)
-  that contains package data with purl (Package URLs) being a first
-  class citizen.
+This purldb consists of these main tools:
+
+- PackageDB that is the reference model (based on ScanCode toolkit) that contains package data with
+  PURL (Package URLs) being a first class citizen.
 - MineCode that contains utilities to mine package repositories
-- MatchCode that contains utilities to index package metadata and resources for
-  matching
+- MatchCode that contains utilities to index package metadata and resources for matching
 - MatchCode.io that provides package matching functionalities for codebases
 - ClearCode that contains utilities to mine Clearlydefined for package data
-- purldb-toolkit CLI utility and library to use the PurlDB, its API and various
-  related libraries.
+- purldb-toolkit CLI utility and library to use the PurlDB, its API and various related libraries.
 
-These are designed to be used first for reference such that one can query for
-packages by purl and validate purl existence.
+These are designed to be used first for reference such that one can query for packages by purl and
+validate purl existence.
 
-In the future, the collected packages will be used as reference for dependency
-resolution, as a reference knowledge base for all package data, as a reference
-for vulnerable range resolution and more.
+In the future, the collected packages will be used as reference for dependency resolution, as a
+reference knowledge base for all package data, as a reference for vulnerable range resolution and
+more.
 
 Documentation
 -------------

azure-pipelines.yml

@@ -39,7 +39,7 @@ jobs:
           image_name: ubuntu-22.04
           python_versions: ['3.10']
           test_suites:
-              all: make check_docs
+              all: make docs check_docs
 
     - template: etc/ci/azure-posix.yml
       parameters:

docs/source/how-to-guides/deploy_to_devel.rst

@@ -13,7 +13,7 @@ directly or through a command line purlcli tool sub-command.
 
 .. note::
     This tutorial assumes that you have a working installation of PurlDB and MatchCode.io
-    If you don't, please refer to the `installation <../purldb/overview.html#installation>`_ page.
+    If you don't, please refer to the `installation <../how-to-guides/installation.html#installation>`_ page.
 
 
 Why mapping binary back to sources?

docs/source/how-to-guides/installation.rst

@@ -1,12 +1,13 @@
+============
 Installation
 ============
 
-This article will detail the steps needed to set up a PurlDB instance with
-MatchCode and package scanning using Docker.
+This article details the steps needed to set up a PurlDB instance with MatchCode and package
+scanning using Docker.
 
-MatchCode.io requires that it is installed and running alongside an instance of
-PurlDB, as it needs direct access to PurlDB's data. This is done by running the
-PurlDB and MatchCode.io services on the same Docker network.
+MatchCode.io requires that it is installed and running alongside an instance of PurlDB, as it needs
+direct access to PurlDB's data. This is done by running the PurlDB and MatchCode.io services on the
+same Docker network.
 
 .. code-block:: console
 

docs/source/how-to-guides/symbols_and_strings.rst

@@ -9,7 +9,7 @@ from codebase resources.
 
 .. note::
     This tutorial assumes that you have a working installation of PurlDB.
-    If you don't, please refer to the `installation <../purldb/overview.html#installation>`_ page.
+    If you don't, please refer to the `installation <../how-to-guides/installation.html#installation>`_ page.
 
 
 Problem

docs/source/index.rst

@@ -1,24 +1,26 @@
-Welcome to PurlDB documentation!
+=========================================
+   Welcome to PurlDB documentation!
 =========================================
 
 
 PurlDB aka. ``Package URL Database`` is a database of software package metadata keyed by Package-URL
 or purl that offers information and indentication services about software packages.
 
-A purl or Package-URL is an attempt to standardize existing approaches to reliably identify and
+A PURL or Package-URL is an attempt to standardize existing approaches to reliably identify and
 locate software packages in general and Free and Open Source Software (FOSS) packages in
 particular.
 
-A purl is a URL string used to identify and locate a software package in a mostly universal and
+A PURL is a URL string used to identify and locate a software package in a mostly universal and
 uniform way across programming languages, package managers, packaging conventions, tools, APIs and
 databases.
 
 Why PurlDB?
 ------------------
 
-Modern software is assembled from 100's or 1000's of FOSS packages: being able to catalog these,
-normalize their metadata, track their versions, licenses and dependencies and being able to locate
-and identify them is essential to healthy, sustainable and secure modern software development.
+Modern software is assembled from 1000's of FOSS packages: being able to catalog these, normalize
+their metadata, track their versions, licenses and dependencies and being able to discover, locate
+and identify them as used in a codebase is essential to healthy, sustainable and secure modern
+software development.
 
 This what PurlDB is all about and it offers:
 
@@ -54,7 +56,7 @@ What's in PurlDB?
 The PurlDB project consists of these main tools:
 
 - PackageDB that is the database and reference model (based on ScanCode toolkit)
-  that contains package data with purl (Package URLs) being a first class citizen and the primaty
+  that contains package data with PURL (Package URLs) being a first class citizen and the primaty
   key to access information.
 
 - MineCode that contains utilities to mine package repositories and populate the PackageDB

docs/source/matchcode/index.rst

@@ -1,5 +1,6 @@
-Matchcode
-==========
+===========
+ Matchcode
+===========
 
 MatchCode.io
 ------------
@@ -8,12 +9,16 @@ MatchCode.io is a Django app, based off of ScanCode.io, that exposes one API
 endpoint, ``api/matching``, which takes a ScanCode.io codebase scan, and
 performs Package matching on it.
 
-Currently, it performs three matching steps:
+Currently, it performs three matching steps, using the PackageDB indices:
 
-  * Match codebase resources against the Packages in the PackageDB
-  * Match codebase resources against the Resources in the PackageDB
-  * Match codebase directories against the directory matching indices of
-    MatchCode
+* Match codebase files against whole Packages archives
+* Match exactly codebase files against files
+* Match codebase directories exactly and approximately against directory indices
+
+Upcoming features include:
+
+* Match codebase files approximately
+* Match codebase file fragments (aka. snippets) including attempting to match AI-generated code
 
 
 MatchCode.io API Endpoints
@@ -28,23 +33,23 @@ MatchCode.io API Endpoints
 Docker Setup for Local Development and Testing
 ----------------------------------------------
 
-PurlDB and MatchCode.io are two separate Django apps. In order to run both of
-these Django apps on the same host, we need to use Traefik.
+PurlDB and MatchCode.io are two separate Django apps. In order to run both of these Django apps on
+the same host, we need to use Traefik.
 
-Traefik is an edge router that receives requests and finds out which services
-are responsible for handling them. In the docker-compose.yml files for PurlDB
-and MatchCode.io, we have made these two services part of the same Docker
-network and set up the routes for each service.
+Traefik is an edge router that receives requests and finds out which services are responsible for
+handling them. In the docker-compose.yml files for PurlDB and MatchCode.io, we have made these two
+services part of the same Docker network and set up the routes for each service.
 
-All requests to the host go to the PurlDB service, but requests that go to the
-``api/matching`` endpoint are routed to the MatchCode.io service.
+All requests to the host go to the PurlDB service, but requests that go to the ``api/matching``
+endpoint are routed to the MatchCode.io service.
 
 To run PurlDB and Matchcode.io with Docker:
 ::
 
   docker compose -f docker-compose.yml up -d
   docker compose -f docker-compose.matchcodeio.yml up -d
 
+
 Scancode.io pipeline
 ---------------------
 

docs/source/matchcode/matchcode-pipeline.rst

@@ -1,13 +1,14 @@
-Code Matching
-=============
+===========================
+  Code Matching pipeline
+===========================
 
 The aim of this tutorial is to show how to use the MatchCode.io API to perform
 code matching on an archive of files.
 
 .. note::
     This tutorial assumes that you have a working installation of PurlDB. If you
     don't, please refer to the `installation
-    <../purldb/overview.html#installation>`_ page.
+    <../how-to-guides/installation.html#installation>`_ page.
 
 Throughout this tutorial, we will use ``pkg:npm/[email protected]`` and a
 modified copy of ``index.js`` from it.

matchcode/api.py

@@ -15,7 +15,6 @@
 from matchcode_toolkit.fingerprinting import create_halohash_chunks
 from matchcode_toolkit.fingerprinting import hexstring_to_binarray
 from matchcode_toolkit.fingerprinting import split_fingerprint
-from matchcode_toolkit.halohash import byte_hamming_distance
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.serializers import CharField
@@ -25,6 +24,7 @@
 from rest_framework.serializers import ReadOnlyField
 from rest_framework.serializers import Serializer
 from rest_framework.viewsets import ReadOnlyModelViewSet
+from samecode.halohash import byte_hamming_distance
 
 from matchcode.models import ApproximateDirectoryContentIndex
 from matchcode.models import ApproximateDirectoryStructureIndex

matchcode/migrations/0003_snippetindex.py

@@ -0,0 +1,53 @@
+# Generated by Django 5.1.2 on 2024-10-30 23:19
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("matchcode", "0002_alter_approximatedirectorycontentindex_package_and_more"),
+        ("packagedb", "0086_alter_party_name"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SnippetIndex",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "fingerprint",
+                    models.BinaryField(
+                        db_index=True,
+                        help_text="Binary form of a snippet fingerprint",
+                        max_length=16,
+                    ),
+                ),
+                (
+                    "package",
+                    models.ForeignKey(
+                        help_text="The Package that this snippet fingerprint is from",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="packagedb.package",
+                    ),
+                ),
+                (
+                    "resource",
+                    models.ForeignKey(
+                        help_text="The Package that this snippet fingerprint is from",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="packagedb.resource",
+                    ),
+                ),
+            ],
+        ),
+    ]