Ensure streaming queries don't have limit clauses (#7382)

* Ensure streaming queries don't impose limit clauses on the sql when called in "synchronous" mode.

* changelog

Author: michaelabuckley

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_8_0/mb-ensure-streaming-sync-does-not-use-limit.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 7382
+title: "When the fetchSizeDefaultMaximum setting was configured, streaming queries would have a limit clause that restricted the results.
+  This has been corrected.  To ensure streaming, use SearchParameterMap.setLoadSynchronous(true) when calling the dao searchForIdStream() method."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -3098,7 +3098,8 @@ private void fetchNext() {
 		}
 
 		private Integer calculateMaxResultsToFetch() {
-			if (myParams.getLoadSynchronousUpTo() != null) {
+			if (myParams.isLoadSynchronous()) {
+				// this might be null - we support this for streaming.
 				return myParams.getLoadSynchronousUpTo();
 			} else if (myParams.getOffset() != null && myParams.getCount() != null) {
 				return myParams.getEverythingMode() != null

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4QueryCountTest.java

@@ -69,6 +69,8 @@
 import ca.uhn.fhir.test.utilities.server.RestfulServerExtension;
 import ca.uhn.fhir.util.BundleBuilder;
 import jakarta.annotation.Nonnull;
+import org.assertj.core.api.Condition;
+import org.assertj.core.data.Index;
 import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IIdType;
@@ -198,22 +200,25 @@ public class FhirResourceDaoR4QueryCountTest extends BaseResourceProviderR4Test
 	@AfterEach
 	public void afterResetDao() {
 		mySubscriptionSettings.clearSupportedSubscriptionTypesForUnitTest();
-		myStorageSettings.setAllowMultipleDelete(new JpaStorageSettings().isAllowMultipleDelete());
-		myStorageSettings.setAutoCreatePlaceholderReferenceTargets(new JpaStorageSettings().isAutoCreatePlaceholderReferenceTargets());
-		myStorageSettings.setAutoVersionReferenceAtPaths(new JpaStorageSettings().getAutoVersionReferenceAtPaths());
-		myStorageSettings.setDeleteEnabled(new JpaStorageSettings().isDeleteEnabled());
-		myStorageSettings.setHistoryCountMode(JpaStorageSettings.DEFAULT_HISTORY_COUNT_MODE);
-		myStorageSettings.setIndexMissingFields(new JpaStorageSettings().getIndexMissingFields());
-		myStorageSettings.setMassIngestionMode(new JpaStorageSettings().isMassIngestionMode());
-		myStorageSettings.setMatchUrlCacheEnabled(new JpaStorageSettings().isMatchUrlCacheEnabled());
-		myStorageSettings.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(new JpaStorageSettings().isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets());
-		myStorageSettings.setResourceClientIdStrategy(new JpaStorageSettings().getResourceClientIdStrategy());
-		myStorageSettings.setResourceMetaCountHardLimit(new JpaStorageSettings().getResourceMetaCountHardLimit());
-		myStorageSettings.setRespectVersionsForSearchIncludes(new JpaStorageSettings().isRespectVersionsForSearchIncludes());
-		myStorageSettings.setTagStorageMode(new JpaStorageSettings().getTagStorageMode());
+
+		JpaStorageSettings defaultStorageSettings = new JpaStorageSettings();
+		myStorageSettings.setAllowMultipleDelete(defaultStorageSettings.isAllowMultipleDelete());
+		myStorageSettings.setAutoCreatePlaceholderReferenceTargets(defaultStorageSettings.isAutoCreatePlaceholderReferenceTargets());
+		myStorageSettings.setAutoVersionReferenceAtPaths(defaultStorageSettings.getAutoVersionReferenceAtPaths());
+		myStorageSettings.setDeleteEnabled(defaultStorageSettings.isDeleteEnabled());
+		myStorageSettings.setHistoryCountMode(defaultStorageSettings.getHistoryCountMode());
+		myStorageSettings.setIndexMissingFields(defaultStorageSettings.getIndexMissingFields());
+		myStorageSettings.setMassIngestionMode(defaultStorageSettings.isMassIngestionMode());
+		myStorageSettings.setMatchUrlCacheEnabled(defaultStorageSettings.isMatchUrlCacheEnabled());
+		myStorageSettings.setPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets(defaultStorageSettings.isPopulateIdentifierInAutoCreatedPlaceholderReferenceTargets());
+		myStorageSettings.setResourceClientIdStrategy(defaultStorageSettings.getResourceClientIdStrategy());
+		myStorageSettings.setResourceMetaCountHardLimit(defaultStorageSettings.getResourceMetaCountHardLimit());
+		myStorageSettings.setRespectVersionsForSearchIncludes(defaultStorageSettings.isRespectVersionsForSearchIncludes());
+		myStorageSettings.setTagStorageMode(defaultStorageSettings.getTagStorageMode());
 		myStorageSettings.setExpungeEnabled(false);
-		myStorageSettings.setUniqueIndexesEnabled(new JpaStorageSettings().isUniqueIndexesEnabled());
-		myStorageSettings.setUniqueIndexesCheckedBeforeSave(new JpaStorageSettings().isUniqueIndexesCheckedBeforeSave());
+		myStorageSettings.setUniqueIndexesEnabled(defaultStorageSettings.isUniqueIndexesEnabled());
+		myStorageSettings.setUniqueIndexesCheckedBeforeSave(defaultStorageSettings.isUniqueIndexesCheckedBeforeSave());
+		myStorageSettings.setFetchSizeDefaultMaximum(defaultStorageSettings.getFetchSizeDefaultMaximum());
 
 		myFhirContext.getParserOptions().setStripVersionsFromReferences(true);
 		TermReadSvcImpl.setForceDisableHibernateSearchForUnitTest(false);
@@ -4613,6 +4618,27 @@ public void testTransactionWithMultiplePreExistingInlineMatchUrls(boolean theMat
 		assertEquals(0, myCaptureQueriesListener.countDeleteQueriesForCurrentThread());
 	}
 
+	@Test
+	void testStreamingQueryDoesNotUseLimit() {
+	    // given
+		myCaptureQueriesListener.clear();
+		myStorageSettings.setFetchSizeDefaultMaximum(100);
+
+		// when
+		Long count = this.runInTransaction(() ->
+			myPatientDao.searchForIdStream(new SearchParameterMap().setLoadSynchronous(true), mySrd, null)
+				.count());
+
+		// then
+		assertEquals(0, count);
+		myCaptureQueriesListener.logSelectQueries();
+		List<SqlQuery> selectQueries = myCaptureQueriesListener.getSelectQueriesForCurrentThread();
+		Condition<SqlQuery> queryNotContainLimit = new Condition<>(query -> !query.getSql(false, false).matches(".*first .* rows.*"), "query does not have limit");
+		assertThat(selectQueries)
+			.hasSize(1)
+				.has(queryNotContainLimit, Index.atIndex(0));
+	}
+
 
 	private void assertQueryCount(int theExpectedSelectCount, int theExpectedUpdateCount, int theExpectedInsertCount, int theExpectedDeleteCount) {