sqlite-utils json dogs.db "select * from dogs"

Author: simonw

docs/cli.rst

@@ -11,11 +11,59 @@ Running queries and returning CSV
 
 You can execute a SQL query against a database and get the results back as CSV like this::
 
-    $ sqlite-utils csv docs.db "select id, title, author from documents"
+    $ sqlite-utils csv dogs.db "select * from dogs"
+    id,age,name
+    1,4,Cleo
+    2,2,Pancakes
 
 This will default to including the column names as a header row. To exclude the headers, use ``--no-headers``::
 
-    $ sqlite-utils csv docs.db "select id, title, author from documents" --no-headers
+    $ sqlite-utils csv dogs.db "select * from dogs" --no-headers
+    1,4,Cleo
+    2,2,Pancakes
+
+Running queries and returning JSON
+==================================
+
+You can execute a SQL query against a database and get the results back as JSON like this::
+
+    $ sqlite-utils json dogs.db "select * from dogs"
+    [{"id": 1, "age": 4, "name": "Cleo"},
+     {"id": 2, "age": 2, "name": "Pancakes"}]
+
+Use ``--nl`` to get back newline-delimited JSON objects::
+
+    $ sqlite-utils json --nl dogs.db "select * from dogs"
+    {"id": 1, "age": 4, "name": "Cleo"}
+    {"id": 2, "age": 2, "name": "Pancakes"}
+
+You can use ``--arrays`` to request ararys instead of objects::
+
+    $ sqlite-utils json --arrays dogs.db "select * from dogs"
+    [[1, 4, "Cleo"],
+     [2, 2, "Pancakes"]]
+
+You can also combine ``--arrays`` and ``--nl``::
+
+    $ sqlite-utils json --arrays --nl dogs.db "select * from dogs"
+    [1, 4, "Cleo"]
+    [2, 2, "Pancakes"]
+
+If you want to pretty-print the output further, you can pipe it through ``python -mjson.tool``::
+
+    $ sqlite-utils json dogs.db "select * from dogs" | python -mjson.tool
+    [
+        {
+            "id": 1,
+            "age": 4,
+            "name": "Cleo"
+        },
+        {
+            "id": 2,
+            "age": 2,
+            "name": "Pancakes"
+        }
+    ]
 
 Listing tables
 ==============
@@ -71,7 +119,6 @@ You can import all three records into an automatically created ``dogs`` table an
 
     $ sqlite-utils insert dogs.db dogs dogs.json --pk=id
 
-
 Upserting data
 ==============
 

sqlite_utils/cli.py

@@ -1,8 +1,10 @@
 import click
 import sqlite_utils
-import json
+from sqlite_utils.utils import iter_pairs
+import json as json_std
 import sys
 import csv as csv_std
+import sqlite3
 
 
 @click.group()
@@ -72,7 +74,7 @@ def optimize(path, no_vacuum):
 def insert(path, table, json_file, pk):
     "Insert records from JSON file into the table, create table if it is missing"
     db = sqlite_utils.Database(path)
-    docs = json.load(json_file)
+    docs = json_std.load(json_file)
     if isinstance(docs, dict):
         docs = [docs]
     db[table].insert_all(docs, pk=pk)
@@ -90,7 +92,7 @@ def insert(path, table, json_file, pk):
 def upsert(path, table, json_file, pk):
     "Upsert records based on their primary key"
     db = sqlite_utils.Database(path)
-    docs = json.load(json_file)
+    docs = json_std.load(json_file)
     if isinstance(docs, dict):
         docs = [docs]
     db[table].upsert_all(docs, pk=pk)
@@ -115,3 +117,42 @@ def csv(path, sql, no_headers):
         writer.writerow([c[0] for c in cursor.description])
     for row in cursor:
         writer.writerow(row)
+
+
+@cli.command()
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("sql")
+@click.option("--nl", help="Output newline-delimited JSON", is_flag=True, default=False)
+@click.option(
+    "--arrays",
+    help="Output rows as arrays instead of objects",
+    is_flag=True,
+    default=False,
+)
+def json(path, sql, nl, arrays):
+    "Execute SQL query and return the results as JSON"
+    db = sqlite_utils.Database(path)
+    cursor = iter(db.conn.execute(sql))
+    # We have to iterate two-at-a-time so we can know if we
+    # should output a trailing comma or if we have reached
+    # the last row.
+    row = None
+    first = True
+    headers = [c[0] for c in cursor.description]
+    for row, next_row, is_last in iter_pairs(cursor):
+        # We now reliably have row and next_row
+        data = row
+        if not arrays:
+            data = dict(zip(headers, row))
+        line = "{firstchar}{serialized}{maybecomma}{lastchar}".format(
+            firstchar=("[" if first else " ") if not nl else "",
+            serialized=json_std.dumps(data),
+            maybecomma="," if (not nl and not is_last) else "",
+            lastchar="]" if (is_last and not nl) else "",
+        )
+        click.echo(line)
+        first = False

sqlite_utils/utils.py

@@ -0,0 +1,28 @@
+def iter_pairs(iterator):
+    # Yields (item, next_item, is_last) pairs
+    # Last row is (item, None, True)
+    first = True
+    next_item = None
+    while True:
+        next_done = False
+        if first:
+            item, done = next_with_done(iterator)
+            if done:
+                break
+            next_item, next_done = next_with_done(iterator)
+            first = False
+        else:
+            item = next_item
+            next_item, next_done = next_with_done(iterator)
+
+        yield item, next_item, next_done
+
+        if next_done:
+            break
+
+
+def next_with_done(iterator):
+    try:
+        return next(iterator), False
+    except StopIteration:
+        return None, True

tests/test_cli.py

@@ -151,3 +151,39 @@ def test_csv(db_path):
         cli.cli, ["csv", db_path, "select id, name, age from dogs", "--no-headers"]
     )
     assert "1,Cleo,4\n2,Pancakes,2\n" == result.output
+
+
+@pytest.mark.parametrize(
+    "args,expected",
+    [
+        (
+            [],
+            '[{"id": 1, "name": "Cleo", "age": 4},\n {"id": 2, "name": "Pancakes", "age": 2}]',
+        ),
+        (
+            ["--nl"],
+            '{"id": 1, "name": "Cleo", "age": 4}\n{"id": 2, "name": "Pancakes", "age": 2}',
+        ),
+        (["--arrays"], '[[1, "Cleo", 4],\n [2, "Pancakes", 2]]'),
+        (["--arrays", "--nl"], '[1, "Cleo", 4]\n[2, "Pancakes", 2]'),
+    ],
+)
+def test_json(db_path, args, expected):
+    db = Database(db_path)
+    with db.conn:
+        db["dogs"].insert_all(
+            [
+                {"id": 1, "age": 4, "name": "Cleo"},
+                {"id": 2, "age": 2, "name": "Pancakes"},
+            ]
+        )
+    result = CliRunner().invoke(
+        cli.cli, ["csv", db_path, "select id, name, age from dogs"]
+    )
+    assert 0 == result.exit_code
+    assert "id,name,age\n1,Cleo,4\n2,Pancakes,2\n" == result.output
+    # Test the no-headers option:
+    result = CliRunner().invoke(
+        cli.cli, ["json", db_path, "select id, name, age from dogs"] + args
+    )
+    assert expected == result.output.strip()