feat: add sourcegraph query.

Author: dudleycarr

.gitignore

@@ -1,2 +1,5 @@
 .env
 .idea
+.task
+data/*.jsonl
+data/*.csv

README.md

@@ -1,7 +1,16 @@
 # non-npm-package-json-files
 Get a collection of package.json files for non-NPM packages
 
-# Requirements
+## Why?
+
+We needed `package.json` files from real projects that aren't packages published to NPM. While
+NPM can tell you the absolute usage of NPM packages in terms of download numbers, we were
+interested in the set of dependencies that people were using together in a given project.
+
+More details about how we used a sample of these package.json files to simulation for StackAid:
+[StackAid in Beta](https://www.stackaid.us/blog/stackaid-in-beta)
+
+## Requirements
 
 * [Brew](https://brew.sh) (MacOSX)
 * [Task](https://taskfile.dev)
@@ -16,7 +25,7 @@ On MacOS:
 brew install brew install go-task/tap/go-task && task brew:requirements
 ```
 
-# Get Sourcegraph Access Token
+## Get a Sourcegraph access token
 
 Use the `src` CLI to see if you're authenticated:
 ```shell
@@ -34,3 +43,43 @@ SRC_ACCESS_TOKEN=<your access token>
 Once configured correctly, rerun `src:login` task to confirm your
 configuration.
 
+## Query Sourcegraph
+
+To query for all package.json files on GitHub that aren't in `node_modules` or directories such
+as `test`, `fixture` or `examples`:
+
+```shell
+task src:query
+```
+
+The command will take about 1 minute and return just over 1M results. The results file in the data
+directory: `./data/src_github_results.jsonl` and it should look like this:
+
+```json lines
+{"type":"path","path":"package.json","repository":"freeCodeCamp/freeCodeCamp","branches":[""],"commit":"382717cce4ea5593eb623ba5ef0bd47c534411d1"}
+{"type":"path","path":"web/package.json","repository":"freeCodeCamp/freeCodeCamp","branches":[""],"commit":"382717cce4ea5593eb623ba5ef0bd47c534411d1"}
+{"type":"path","path":"curriculum/package.json","repository":"freeCodeCamp/freeCodeCamp","branches":[""],"commit":"382717cce4ea5593eb623ba5ef0bd47c534411d1"}
+{"type":"path","path":"tools/crowdin/package.json","repository":"freeCodeCamp/freeCodeCamp","branches":[""],"commit":"382717cce4ea5593eb623ba5ef0bd47c534411d1"}
+{"type":"path","path":"tools/scripts/seed/package.json","repository":"freeCodeCamp/freeCodeCamp","branches":[""],"commit":"382717cce4ea5593eb623ba5ef0bd47c534411d1"}
+```
+
+To convert the file to a CSV:
+
+```shell
+task src:query:csv
+```
+
+The results will be in `./data/src_github_results.csv` and it should looks this this:
+
+```csv
+repo,commit_sha,path
+freeCodeCamp/freeCodeCamp,382717cce4ea5593eb623ba5ef0bd47c534411d1,package.json
+freeCodeCamp/freeCodeCamp,382717cce4ea5593eb623ba5ef0bd47c534411d1,web/package.json
+freeCodeCamp/freeCodeCamp,382717cce4ea5593eb623ba5ef0bd47c534411d1,curriculum/package.json
+freeCodeCamp/freeCodeCamp,382717cce4ea5593eb623ba5ef0bd47c534411d1,tools/crowdin/package.json
+freeCodeCamp/freeCodeCamp,382717cce4ea5593eb623ba5ef0bd47c534411d1,tools/scripts/seed/package.json
+```
+
+## Debug Sourcegraph query
+
+Try the [query](https://sourcegraph.com/search?q=context:global+file:%28%5E%7C/%29package.json%24+fork:no+-file:%28%5E%7C/%29%5C.+-file:%28%5E%7C/%29%28node_modules%7Ctest%7Ctests%7Cfixture%7Cfixtures%7Cexamples%29/+count:all+archived:no+-file:%28%5E%7C/%29vendor/+&patternType=standard) on Sourcegraph!

Taskfile.yaml

@@ -2,11 +2,17 @@
 
 version: '3'
 
+vars:
+  DATA_DIR: ./data
+  SRC_RESULTS_JSONL: "{{.DATA_DIR}}/src_github_results.jsonl"
+  SRC_RESULTS_CSV: "{{.DATA_DIR}}/src_github_results.csv"
+
 dotenv:
   - .env
 
 tasks:
   brew:requirements:
+    desc: Install required utilities.
     cmds:
       - |-
         brew install \
@@ -16,5 +22,48 @@ tasks:
           sourcegraph/src-cli/src-cli \
           sqlite \
           xsv
+
   src:login:
-    - src login
+    desc: Test Sourcegraph CLI authentication.
+    cmds:
+      - src login
+
+  src:query:
+    desc: Query Sourcegraph for package.json files
+    summary: |
+      Query SourceGraph for package.json files.
+
+      SourceGraph query asks for all package.json files excluding files found in directories such
+      as node_modules, test, fixture, and examples. The returned results are filtered to contain
+      GitHub repositories and reformatting the repository field in the output.
+    cmds:
+      - |-
+        src search -stream -json '{{ .SRC_QUERY }}' \
+          | jq -c 'select(.type == "path") | select(.repository | test("^github.com"))' \
+          | jq -c '.repository = (.repository | sub("github.com/"; ""))' \
+          > {{ .SRC_RESULTS_JSONL }}
+    vars:
+      SRC_QUERY: >-
+        file:(^|/)package.json$
+        fork:no
+        archived:no
+        -file:(^|/)\.
+        -file:(^|/)(node_modules|test|tests|fixture|fixtures|examples|vendor)/
+        count:all
+    generates:
+      - "{{ .SRC_RESULTS_JSONL }}"
+
+  src:query:csv:
+    desc: Convert Sourcegraph query results into a CSV.
+    summary: |
+      Convert Sourcegraph query results into a CSV.
+    cmds:
+      - echo "repo,commit_sha,path" > {{ .SRC_RESULTS_CSV }}
+      - |-
+        jq -r '[.repository, .commit, .path] | @csv' {{ .SRC_RESULTS_JSONL }} \
+          | xsv fmt \
+          >> {{ .SRC_RESULTS_CSV }}
+    sources:
+      - "{{ .SRC_RESULTS_JSONL }}"
+    generates:
+      - "{{ .SRC_RESULTS_CSV }}"