initial commit

Author: TylerBrock

.gitignore

@@ -0,0 +1 @@
+node_modules

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Hustle, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,50 @@
+## StitchQuery - "for getting all your stuff"™
+
+StitchQuery allows you to query for more than 1000 objects by making multiple requests and stitching
+the results together.
+
+If you need more than 10,000 results (unlikely), you can use a SuperStitchQuery.
+
+#### How does it work?
+
+StitchQuery sets the maximum hosted Parse limit of 1000 on the provided query and makes an initial
+request for the data. If 1000 objects are returned (the maximum batch size) we make additional
+requests, taking care to increment the query's skip by the batch size of 1000 until we reach the
+maximum skip (10k). This allows you to fetch up to 10,000 results using StitchQuery.
+
+If desired, a SuperStitchQuery can be performed which will fetch unlimited results. It does this by
+applying a sort on `createdAt` to the provided query and, upon reaching the maximum skip limit, gets
+the date of the final item retrieved. To fetch the next batch it adds an additional query predicate
+requesting only items that have `createdAt` greater than that of the last item returned and resets
+the skip to zero. This process is repeated until all results are retrieved (receiving a batch with
+less than the requested limit of 1000 documents).
+
+ - Note: SuperStitchQuery should **not** be used on queries that already have a sort applied!
+
+#### Installation
+```sh
+npm install --save parse-stitch-query
+```
+
+#### Usage w/ Parse
+```js
+var Parse =  require('parse-shim');
+var StitchQuery = require('parse-stitch-query');
+
+var query = new Parse.Query(Model);
+query.equalTo('blah', 'cool');
+
+// Get up to 10,000 results
+StitchQuery(query).then((results) => {
+  // all the results are here!
+}, (error) => {
+  // oh noes! errors
+})
+
+// SuperStitchQuery, for unlimited results
+StitchQuery(query, true).then((results) => {
+  // all the results are here!
+}, (error) => {
+  // oh noes! errors
+})
+```

index.js

@@ -0,0 +1,70 @@
+'use strict';
+
+const Parse = require('parse-shim');
+
+//
+// Run the specified parse query multiple times, modifying skip/limit each
+// time to download all available objects. Return a promise with the stitched
+// results from all individual queries. This works around the limit of 1000
+// results per parse query.
+//
+// For safety, unless superStitch is specified, this limits the total number
+// of queries to 10, yielding an effective max of 10,000 results.
+//
+// If 'superStitch' is specified, results are ordered and additionally paged
+// by the 'createdAt' date field after exhausting the 10k skip. This effectively
+// allows a query to yield unlimited results.
+//
+// @param query {Parse.Query} The query to stitch.
+// @param superStitch {Boolean} If you want more than 10k rows you can superStitch.
+//
+// NOTE: superStitch is not suitable for use with a sort (as it applies its own sort on createdAt)
+// NOTE: This will modify the specified query, do not reuse the query after calling this function.
+//
+function StitchQuery(query, superStitch) {
+  const pageSize = 1000;
+  const maxQueries = 10;
+
+  query.limit(pageSize);
+
+  if (superStitch) {
+    query.ascending('createdAt');
+  }
+
+  const stitchedPromise = new Parse.Promise();
+  const stitchedResults = [];
+  function getNextPage(curPage, startDate) {
+    query.skip(curPage * pageSize);
+
+    if (startDate) {
+      query.greaterThan('createdAt', startDate);
+    }
+
+    const pagePromise = query.find();
+
+    pagePromise.done((pageResults) => {
+      pageResults.forEach(result => { stitchedResults.push(result); });
+      const maxBatchSize = pageResults.length === pageSize;
+      const lastResult = stitchedResults[stitchedResults.length - 1];
+
+      if (maxBatchSize) {
+        if (curPage + 1 < maxQueries) {
+          getNextPage(curPage + 1, startDate);
+        } else if (superStitch) {
+          getNextPage(0, lastResult.createdAt);
+        } else {
+          stitchedPromise.resolve(stitchedResults);
+        }
+      } else {
+        stitchedPromise.resolve(stitchedResults);
+      }
+
+    });
+    pagePromise.fail(stitchedPromise.reject);
+  };
+
+  getNextPage(0);
+  return stitchedPromise;
+};
+
+module.exports = StitchQuery;

package.json

@@ -0,0 +1,36 @@
+{
+  "name": "parse-stitch-query",
+  "version": "1.0.0",
+  "description": "Query helper for retrieving a large number of objects from Parse",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha test.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/HustleInc/parse-stitch-query.git"
+  },
+  "keywords": [
+    "Parse",
+    "query",
+    "limit"
+  ],
+  "author": "Tyler Brock, Mike Mintz",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/HustleInc/parse-stitch-query/issues"
+  },
+  "homepage": "https://github.com/HustleInc/parse-stitch-query#readme",
+  "dependencies": {
+    "mocha": "2.4.5",
+    "parse": "^1.7.0",
+    "parse-shim": "1.0.6"
+  },
+  "devDependencies": {
+    "chai": "3.5.0",
+    "parse-mockdb": "0.1.12"
+  },
+  "engines": {
+    "node": ">=4.0.0"
+  }
+}

test.js

@@ -0,0 +1,95 @@
+'use strict';
+
+const Parse = require('parse-shim');
+
+const ParseMockDB = require('parse-mockdb');
+const StitchQueryP = require('./index');
+const expect = require('chai').expect;
+
+class Obj extends Parse.Object {
+  constructor() {
+    super('Obj');
+  }
+};
+
+function saveObjsP(count) {
+  const objsToSave = [];
+  for (let i=0; i<count; i++) {
+    const obj = new Obj();
+    objsToSave.push(obj);
+  }
+  return Parse.Object.saveAll(objsToSave);
+}
+
+describe('StitchQueryP', function() {
+  beforeEach(function() {
+    Parse.MockDB.mockDB();
+  });
+
+  afterEach(function() {
+    Parse.MockDB.cleanUp();
+  });
+
+  context('when we have exactly 100 objects in collection', function() {
+    it('returns all of the expected results', function() {
+      return saveObjsP(100).then((objs) => {
+        return StitchQueryP(new Parse.Query(Obj))
+      }).then((results) => {
+        expect(results).to.have.lengthOf(100);
+      })
+    });
+  });
+
+  context('when we have exactly 1000 objects in collection', function() {
+    it('returns all of the expected results', function() {
+      return saveObjsP(1000).then((objs) => {
+        return StitchQueryP(new Parse.Query(Obj))
+      }).then((results) => {
+        expect(results).to.have.lengthOf(1000);
+      })
+    });
+  });
+
+  context('when we have more than 1000 objects in collection', function() {
+    it('returns all of the expected results', function() {
+      return saveObjsP(1001).then((objs) => {
+        return StitchQueryP(new Parse.Query(Obj))
+      }).then((results) => {
+        expect(results).to.have.lengthOf(1001);
+      })
+    });
+  });
+
+  context('when we have exactly 10000 objects in collection', function() {
+    it('returns all of the expected results @slow', function() {
+      this.timeout(30000);
+      return saveObjsP(10000).then((objs) => {
+        return StitchQueryP(new Parse.Query(Obj))
+      }).then((results) => {
+        expect(results).to.have.lengthOf(10000);
+      })
+    });
+  });
+
+  context('when we have greater than 10000 objects in collection', function() {
+    it('returns returns only 10k of the expected results @slow', function() {
+      this.timeout(30000);
+      return saveObjsP(10100).then((objs) => {
+        return StitchQueryP(new Parse.Query(Obj))
+      }).then((results) => {
+        expect(results).to.have.lengthOf(10000);
+      })
+    });
+
+    context('using super stitch query', function() {
+      it('returns all of the expected results @slow', function() {
+        this.timeout(30000);
+        return saveObjsP(10100).then((objs) => {
+          return StitchQueryP(new Parse.Query(Obj), true)
+        }).then((results) => {
+          expect(results).to.have.lengthOf(10100);
+        })
+      });
+    });
+  })
+});