Fix Bug #3115: Add further driveId validation when processing API response data for a OneDrive Personal Shared Folder (#3116)

* Remove 'sea8cc6beffdb43d7976fbc7da445c639' check
* Add further personal account driveId length checks when generating a /delta response for a Shared Folder
* Fix selectRemoteTypeByRemoteDriveId() function to also use the remote item id so that the correct DB record is responded with. This ensures that 'localPathExtension =' reflects the correct local path extension and not some other incorrect local path
* If the 'shared' JSON structure is missing, add a debug message
* Add further checks for invalid 15 character driveId when creating Shared Folder references
* Ensure that the 'driveId' value is correctly fetched and used due to 'UPPERCASE' and 'lowercase' values in API JSON response
* Increase DB version to ensure all old records in the DB are purged
* Sanitise JSON output for debug logging when enabled
* Simplify isValidUTF8Timestamp()
* Debug logging is not correct - missing all single --verbose entries which should be also included when performing a debug capture.
* Based on application debug logs, if /delta fails to send Shared Folders, and the client goes to create these online, we know that these then exist online and are a Shared Folder. Handle in a similar manner to OneDrive Business Shared Folders logic to create the required database tie records if the API fails to send that data originally.
* Remove double call to selectiveSync.isFileNameExcluded
* Align invalid UTF-8 message to be consistent with other UTF-8 failure messages

This PR specifically works around the Microsoft OneDrive change:
* Microsoft moving all OneDrive Personal Accounts to a new backend platform. This is the 'sea8cc6beffdb43d7976fbc7da445c639' change: OneDrive/onedrive-api-docs#1890
* Microsoft failing to provide the Graph API data in the /delta call for accounts moved to the new backend platform: OneDrive/onedrive-api-docs#1891

Author: abraunegg

src/itemdb.d

@@ -225,7 +225,7 @@ Item makeDatabaseItem(JSONValue driveItem) {
 
 final class ItemDatabase {
 	// increment this for every change in the db schema
-	immutable int itemDatabaseVersion = 15;
+	immutable int itemDatabaseVersion = 16;
 
 	Database db;
 	string insertItemStmt;
@@ -371,7 +371,7 @@ final class ItemDatabase {
 			SELECT *
 			FROM item
 			WHERE type = 'remote'
-			AND remoteDriveId = ?1
+			AND remoteDriveId = ?1 AND remoteId = ?2
 		";
 		selectItemByParentIdStmt = "SELECT * FROM item WHERE driveId = ? AND parentId = ?";
 		deleteItemByIdStmt = "DELETE FROM item WHERE driveId = ? AND id = ?";
@@ -633,13 +633,14 @@ final class ItemDatabase {
 		}
 	}
 
-	// This should return the 'remote' DB entry for the given 'remoteDriveId'
-	bool selectRemoteTypeByRemoteDriveId(const(char)[] entryName, out Item item) {
+	// This should return the 'remote' DB entry for the given 'remoteDriveId' and 'remoteId'
+	bool selectRemoteTypeByRemoteDriveId(const(char)[] remoteDriveId, const(char)[] remoteId, out Item item) {
 		synchronized(databaseLock) {
 			auto p = db.prepare(selectRemoteTypeByRemoteDriveIdStmt);
 			scope(exit) p.finalise(); // Ensure that the prepared statement is finalised after execution.
 			try {
-				p.bind(1, entryName);
+				p.bind(1, remoteDriveId);
+				p.bind(2, remoteId);
 				auto r = p.exec();
 				if (!r.empty) {
 					item = buildItem(r);

src/main.d

@@ -151,8 +151,9 @@ int main(string[] cliArgs) {
 	}
 
 	// Determine the application logging verbosity
+	// - As these flags are used to reduce application processing when not required, specifically in a 'debug' scenario, both verboseLogging and debugLogging need to be enabled
 	if (verbosityCount == 1) { verboseLogging = true;} // set __gshared bool verboseLogging in log.d
-	if (verbosityCount >= 2) { debugLogging = true;}   // set __gshared bool debugLogging in log.d
+	if (verbosityCount >= 2) { verboseLogging = true; debugLogging = true;}   // set __gshared bool verboseLogging & debugLogging in log.d
 
 	// Initialize the application logging class, as we know the application verbosity level
 	// If we need to enable logging to a file, we can only do this once we know the application configuration which is done slightly later on

src/onedrive.d

@@ -572,9 +572,18 @@ class OneDriveApi {
 	//   After you have finished receiving all the changes, you may apply them to your local state. To check for changes in the future, call delta again with the @odata.deltaLink from the previous successful response.
 	JSONValue getChangesByItemId(string driveId, string id, string deltaLink) {
 		string[string] requestHeaders;
-		// If Business Account add Prefer: Include-Feature=AddToOneDrive
-		if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) {
+		
+		// From March 1st 2025, this needs to be added to ensure that Shared Folders are sent in the Delta Query Response
+		if (appConfig.accountType == "personal") {
+			// OneDrive Personal Account
 			addIncludeFeatureRequestHeader(&requestHeaders);
+		} else {
+			// Business or SharePoint Library
+			// Only add if configured to do so
+			if (appConfig.getValueBool("sync_business_shared_items")) {
+				// Feature enabled, add headers
+				addIncludeFeatureRequestHeader(&requestHeaders);
+			}
 		}
 
 		string url;
@@ -586,15 +595,26 @@ class OneDriveApi {
 		} else {
 			url = deltaLink;
 		}
+		
+		// get the response
 		return get(url, false, requestHeaders);
 	}
 
 	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_list_children
 	JSONValue listChildren(string driveId, string id, string nextLink) {
 		string[string] requestHeaders;
-		// If Business Account add addIncludeFeatureRequestHeader() which should add Prefer: Include-Feature=AddToOneDrive
-		if ((appConfig.accountType != "personal") && ( appConfig.getValueBool("sync_business_shared_items"))) {
+		
+		// From March 1st 2025, this needs to be added to ensure that Shared Folders are sent in the Delta Query Response
+		if (appConfig.accountType == "personal") {
+			// OneDrive Personal Account
 			addIncludeFeatureRequestHeader(&requestHeaders);
+		} else {
+			// Business or SharePoint Library
+			// Only add if configured to do so
+			if (appConfig.getValueBool("sync_business_shared_items")) {
+				// Feature enabled, add headers
+				addIncludeFeatureRequestHeader(&requestHeaders);
+			}
 		}
 
 		string url;
@@ -819,7 +839,14 @@ class OneDriveApi {
 
 	// Private OneDrive API Functions
 	private void addIncludeFeatureRequestHeader(string[string]* headers) {
-		if (debugLogging) {addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled", ["debug"]);}
+		if (appConfig.accountType == "personal") {
+			// Add logging message for OneDrive Personal Accounts
+			if (debugLogging) {addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header for OneDrive Personal Account Type", ["debug"]);}
+		} else {
+			// Add logging message for OneDrive Business Accounts
+			if (debugLogging) {addLogEntry("Adding 'Include-Feature=AddToOneDrive' API request header as 'sync_business_shared_items' config option is enabled", ["debug"]);}
+		}
+		// Add feature to request headers
 		(*headers)["Prefer"] = "Include-Feature=AddToOneDrive";
 	}