diff --git a/Data/DataPreferences.swift b/Data/DataPreferences.swift index 7b399eb1a..27b38466c 100644 --- a/Data/DataPreferences.swift +++ b/Data/DataPreferences.swift @@ -16,6 +16,8 @@ extension Preferences { /// To count as a successful, the entire process must succeed: fetching, resolving, insertion/update, and save static let lastFetchTimestamp = Option(key: "sync.last-fetch-timestamp", default: 0) + static let lastPreferenceFetchTimestamp = Option(key: "sync.last-fetch-timestamp-device", default: 0) + /// We store this value to signify that a group has been joined /// This is _only_ used on a re-installation to know that the app was deleted and re-installed /// Real Sync seed is stored in a keychain, although preference name of this option is used for both. diff --git a/Data/sync/Sync.swift b/Data/sync/Sync.swift index eb66928b6..9200179e9 100644 --- a/Data/sync/Sync.swift +++ b/Data/sync/Sync.swift @@ -46,8 +46,12 @@ public enum SyncRecordType: String { return map[self] } - var syncFetchMethod: String { - return self == .devices ? "fetch-sync-devices" : "fetch-sync-records" + /// Preferences(devices) have separate fetch timestamp. + var lastFetchTimeStamp: Int { + switch self { + case .prefs: return Preferences.Sync.lastPreferenceFetchTimestamp.value + default: return Preferences.Sync.lastFetchTimestamp.value + } } } @@ -98,6 +102,9 @@ public class Sync: JSInjector { return syncSeed != nil } + let syncFetchMethod = "fetch-sync-records" + let fetchInterval: TimeInterval = 60 + fileprivate var fetchTimer: Timer? /// If sync initialization fails, we should inform user and remove all partial sync setup that happened. @@ -146,7 +153,6 @@ public class Sync: JSInjector { private var syncFetchedHandlers = [() -> Void]() private var baseSyncOrder: String { return Preferences.Sync.baseSyncOrder.value } - private var lastSyncTimestamp: Int { return Preferences.Sync.lastFetchTimestamp.value } override init() { super.init() @@ -179,8 +185,8 @@ public class Sync: JSInjector { Device.deleteAll(context: .existing(context)) } - lastFetchedRecordTimestamp = 0 Preferences.Sync.lastFetchTimestamp.reset() + Preferences.Sync.lastPreferenceFetchTimestamp.reset() lastFetchWasTrimmed = false syncReadyLock = false isSyncFullyInitialized = (false, false, false, false, false, false, false, false) @@ -269,11 +275,6 @@ public class Sync: JSInjector { return seed?.count == Sync.SeedByteLength ? seed : nil } - // TODO: Abstract into classes as static members, each object type needs their own sync time stamp! - // This includes just the last record that was fetched, used to store timestamp until full process has been completed - // then set into defaults - fileprivate(set) var lastFetchedRecordTimestamp: Int? = 0 - // Same abstraction note as above // Used to know if data on get-existing-objects was trimmed, this value is used inside resolved-sync-records fileprivate var lastFetchWasTrimmed: Bool = false @@ -315,15 +316,15 @@ public class Sync: JSInjector { self.fetchWrapper() // Fetch timer to run on regular basis - self.fetchTimer = Timer.scheduledTimer(timeInterval: 30.0, target: self, selector: #selector(Sync.fetchWrapper), userInfo: nil, repeats: true) + self.fetchTimer = Timer.scheduledTimer(timeInterval: self.fetchInterval, target: self, selector: #selector(Sync.fetchWrapper), userInfo: nil, repeats: true) } } // Just throw by itself, does not need to recover or retry due to lack of importance - self.fetch(type: .devices) + self.fetch(type: .prefs) // Use proper variable and store in defaults - if lastSyncTimestamp == 0 { + if Preferences.Sync.lastFetchTimestamp.value == 0 { // Sync local bookmarks, then proceed with fetching // Pull all local bookmarks and update their order with newly aquired device id and base sync order. DataController.perform { context in @@ -361,7 +362,7 @@ public class Sync: JSInjector { // This can be removed and fetch called directly via scheduledTimerBlock @objc func fetchWrapper() { self.fetch(type: .bookmark) - self.fetch(type: .devices) + self.fetch(type: .prefs) } } @@ -445,7 +446,7 @@ extension Sync { executeBlockOnReady() { // Pass in `lastFetch` to get records since that time - let evaluate = "callbackList['\(type.syncFetchMethod)'](null, ['\(type.rawValue)'], \(self.lastSyncTimestamp), \(Sync.RecordFetchAmount))" + let evaluate = "callbackList['\(self.syncFetchMethod)'](null, ['\(type.rawValue)'], \(type.lastFetchTimeStamp), \(Sync.RecordFetchAmount))" self.webView.evaluateJavaScript(evaluate, completionHandler: { (result, error) in completion?(error) @@ -505,28 +506,26 @@ extension Sync { } log.debug("\(fetchedRecords.count) \(recordType.rawValue) processed") - // Make generic when other record types are supported - if recordType != .bookmark { - // Currently only support bookmark timestamp, so do not want to adjust that - return - } + if fetchedRecords.isEmpty { return } // After records have been written, without issue, save timestamp // We increment by a single millisecond to make sure we don't re-fetch the same duplicate records over and over // If there are more records with the same timestamp than the batch size, they will be dropped, // however this is unimportant, as it actually prevents an infinitely recursive loop, of refetching the same records over // and over again - if let stamp = self.lastFetchedRecordTimestamp { - Preferences.Sync.lastFetchTimestamp.value = stamp + 1 - } - - if self.lastFetchWasTrimmed { - // Do fast refresh, do not wait for timer - self.fetch(type: .bookmark) - self.lastFetchWasTrimmed = false + if recordType == .bookmark { + Preferences.Sync.lastFetchTimestamp.value += 1 + + if self.lastFetchWasTrimmed { + // Do fast refresh, do not wait for timer + self.fetch(type: .bookmark) + self.lastFetchWasTrimmed = false + } + + syncFetchedHandlers.forEach { $0() } + } else if recordType == .prefs { + Preferences.Sync.lastPreferenceFetchTimestamp.value += 1 } - - syncFetchedHandlers.forEach { $0() } } func deleteSyncUser(_ data: [String: AnyObject]) { @@ -554,39 +553,45 @@ extension Sync { let ids = fetchedRecords.map { $0.objectId }.compactMap { $0 } - var localbookmarks: [Bookmark]? + var localRecord: [Syncable]? DataController.perform { context in - localbookmarks = recordType.coredataModelType?.get(syncUUIDs: ids, context: context) as? [Bookmark] + localRecord = recordType.coredataModelType?.get(syncUUIDs: ids, context: context) as? [Syncable] - var matchedBookmarks = [[Any]]() + var matchedRecords = [[Any]]() let deviceId = Device.currentDevice(context: context)?.deviceId - for fetchedBM in fetchedRecords { - var localBM: Any = "null" + for fetchedRecord in fetchedRecords { + var local: Any = "null" - if let found = localbookmarks?.find({ $0.syncUUID != nil && $0.syncUUID == fetchedBM.objectId }) { - localBM = found.asDictionary(deviceId: deviceId, action: fetchedBM.action) + if let found = localRecord?.find({ $0.syncUUID != nil && $0.syncUUID == fetchedRecord.objectId }) { + local = found.asDictionary(deviceId: deviceId, action: fetchedRecord.action) } - matchedBookmarks.append([fetchedBM.dictionaryRepresentation(), localBM]) + matchedRecords.append([fetchedRecord.dictionaryRepresentation(), local]) } - /* Top level keys: "bookmark", "action","objectId", "objectData:bookmark","deviceId" */ - // TODO: Check if parsing not required - guard let serializedData = JSONSerialization.jsObject(withNative: matchedBookmarks as AnyObject, escaped: false) else { + guard let serializedData = JSONSerialization.jsObject(withNative: matchedRecords as AnyObject, escaped: false) else { log.error("Critical error: could not serialize data for resolve-sync-records") return } + let lastFetchTimestamp = data?.lastFetchedTimestamp ?? 0 // Only currently support bookmarks, this data will be abstracted (see variable definition note) if recordType == .bookmark { - // Store the last record's timestamp, to know what timestamp to pass in next time if this one does not fail - self.lastFetchedRecordTimestamp = data?.lastFetchedTimestamp - log.info("sync fetched last timestamp \(self.lastFetchedRecordTimestamp ?? 0)") + if lastFetchTimestamp > Preferences.Sync.lastFetchTimestamp.value { + Preferences.Sync.lastFetchTimestamp.value = lastFetchTimestamp + log.info("sync bookmark last timestamp \(lastFetchTimestamp)") + } self.lastFetchWasTrimmed = data?.isTruncated ?? false - } + } else if recordType == .prefs { + + if lastFetchTimestamp > Preferences.Sync.lastPreferenceFetchTimestamp.value { + Preferences.Sync.lastPreferenceFetchTimestamp.value = lastFetchTimestamp + log.info("sync preference last timestamp \(lastFetchTimestamp)") + } + } DispatchQueue.main.async { self.webView.evaluateJavaScript("callbackList['resolve-sync-records'](null, '\(recordType.rawValue)', \(serializedData))", diff --git a/package-lock.json b/package-lock.json index 6ee9f43fb..07d6194cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,9 +64,9 @@ "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==" }, "@types/node": { - "version": "10.14.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.15.tgz", - "integrity": "sha512-CBR5avlLcu0YCILJiDIXeU2pTw7UK/NIxfC63m7d7CVamho1qDEzXKkOtEauQRPMy6MI8mLozth+JJkas7HY6g==" + "version": "10.14.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.17.tgz", + "integrity": "sha512-p/sGgiPaathCfOtqu2fx5Mu1bcjuP8ALFg4xpGgNkcin7LwRyzUKniEHBKdcE1RPsenq5JVPIpMTJSygLboygQ==" }, "accepts": { "version": "1.3.7", @@ -275,9 +275,9 @@ "dev": true }, "aws-sdk": { - "version": "2.505.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.505.0.tgz", - "integrity": "sha512-VSPSUl1xt0K+YAjNGbrNOOJ/q6X9GBIeg+vzxBBy4oX9SkZitKVuvIB6uc4fio9XQkYKvCdl8V/vMsQWyIvVgw==", + "version": "2.521.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.521.0.tgz", + "integrity": "sha512-YRViGahZoHq0lEsy0Cq7bqaLzP8OTzjg35Hz08Rv88ltk+b5UWVpDgIWls7mXCBNTBTYjbfbeGXzNaHJBLKlpA==", "requires": { "buffer": "4.9.1", "events": "1.1.1", @@ -288,6 +288,13 @@ "url": "0.10.3", "uuid": "3.3.2", "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } } }, "aws-sign2": { @@ -1151,7 +1158,7 @@ } }, "brave-sync": { - "version": "github:brave/sync#76bf8f1295b46a7112756af631a8f5cd217953e6", + "version": "github:brave/sync#16a888f180d3ce38fbec9aa1a5215e8f11692601", "from": "github:brave/sync#master", "requires": { "@protobufjs/utf8": "^1.1.0", @@ -1633,9 +1640,9 @@ }, "dependencies": { "buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", - "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==", + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.4.2.tgz", + "integrity": "sha512-iy9koArjAFCzGnx3ZvNA6Z0clIbbFgbdWQ0mKD3hO0krOrZh8UgA6qMKcZvwLJxS+D6iVR76+5/pV56yMNYTag==", "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" @@ -1763,9 +1770,25 @@ "dev": true }, "deep-equal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.0.tgz", + "integrity": "sha512-ZbfWJq/wN1Z273o7mUSjILYqehAktR2NVoSrOukDkU9kg2v/Uv89yU4Cvz8seJeAmtN5oqiefKq8FPuXOboqLw==", + "requires": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "^1.0.12" + } }, "define-property": { "version": "2.0.2", @@ -2994,6 +3017,11 @@ } } }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "get-caller-file": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", @@ -3081,6 +3109,14 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, "has-ansi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", @@ -3283,6 +3319,11 @@ "kind-of": "^3.0.2" } }, + "is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==" + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -3322,6 +3363,11 @@ "kind-of": "^3.0.2" } }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" + }, "is-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", @@ -3398,6 +3444,14 @@ "isobject": "^3.0.1" } }, + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "requires": { + "has": "^1.0.1" + } + }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", @@ -4024,6 +4078,16 @@ } } }, + "object-is": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.0.1.tgz", + "integrity": "sha1-CqYOyZiaCz7Xlc9NBvYs8a1lObY=" + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, "object-visit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", @@ -4531,6 +4595,14 @@ "safe-regex": "^1.1.0" } }, + "regexp.prototype.flags": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.2.0.tgz", + "integrity": "sha512-ztaw4M1VqgMwl9HlPpOuiYgItcHlunW0He2fE6eNfT6E/CF2FtYi9ofOYe4mKntstYk0Fyh/rDRBdS3AnxjlrA==", + "requires": { + "define-properties": "^1.1.2" + } + }, "regexpu-core": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz", @@ -5524,9 +5596,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" }, "validate-npm-package-license": { "version": "3.0.4",