Commit 95ad73fd authored by Amirjanyan's avatar Amirjanyan
Browse files

Merge branch 'file-sync' into 'main'

Implement file syncing with Pod

See merge request !41
parents d54d9f57 0435c493
Pipeline #3242 canceled with stage
......@@ -32,7 +32,7 @@ enum PodAPIPayload {
}
struct FileSHA: Encodable {
var sha: String
var sha256: String
}
struct itemId: Encodable {
var uid: StringUUID
......
......@@ -229,7 +229,7 @@ extension PodAPIStandardRequest {
extension PodAPIDownloadRequest {
static func downloadFile(fileSHAHash: String, fileUID: String) -> PodAPIDownloadRequest where Payload == PodAPIPayload.FileSHA {
PodAPIDownloadRequest(path: "get_file", payload: PodAPIPayload.FileSHA(sha: fileSHAHash), fileUID: fileUID)
PodAPIDownloadRequest(path: "get_file", payload: PodAPIPayload.FileSHA(sha256: fileSHAHash), fileUID: fileUID)
}
}
......
......@@ -125,6 +125,7 @@ class DatabaseController {
/// iOS only (used for syncing decisions
t.column("syncState", .text).defaults(to: SyncState.create).notNull(onConflict: .replace)
t.column("fileState", .text).defaults(to: FileState.skip).notNull(onConflict: .replace)
t.column("syncHasPriority", .boolean).defaults(to: false).notNull(onConflict: .replace)
}
try db.create(index: "idx_items_id", on: "items", columns: ["id"], unique: true)
......
......@@ -130,11 +130,15 @@ enum DemoData {
for item in processedItems {
let record = ItemRecord(id: item.id, type: item.type, dateCreated: item.dateCreated, dateModified: item.dateModified, dateServerModified: item.dateServerModified)
if record.type == "File" {
record.fileState = .needsUpload
}
try record.insert(db)
let id = item.tempID ?? item.id.uid
if let rowId = record.rowId {
rowIdLookup[id] = rowId
}
}
for item in processedItems {
......@@ -238,8 +242,27 @@ enum DemoData {
continue
}
do {
let result = try PropertyDatabaseValue(value: propertyValue, propertyType: expectedType.valueType, debugInfo: "\(itemType).\(propertyName)")
properties.append(DemoData_Property(name: propertyName, value: result))
var databaseValue = try PropertyDatabaseValue(value: propertyValue, propertyType: expectedType.valueType, debugInfo: "\(itemType).\(propertyName)")
switch ((itemType, propertyName, databaseValue)) {
case ("File", "filename", let .string(fileName)) :
let newFileName = "\(UUID().uuidString).\(fileName.fileExtension ?? "jpg")"
let url = FileStorageController.getFileStorageURL().appendingPathComponent(newFileName)
guard let demoDirectory = Bundle.main.resourceURL?.appendingPathComponent("demoAssets", isDirectory: true) else {
continue
}
let sourcePath = demoDirectory.appendingPathComponent("\(fileName.fileName ?? "").\(fileName.fileExtension ?? "jpg")")
try FileManager.default.copyItem(atPath: sourcePath.path, toPath: url.path)
// Also add sha256 property for item
let sha256 = try FileHelper.getSHAHash(url: url)
properties.append(DemoData_Property(name: "sha256", value: .string(sha256)))
databaseValue = .string(newFileName)
default:
break
}
properties.append(DemoData_Property(name: propertyName, value: databaseValue))
}
catch {
try handleError(error.localizedDescription)
......
......@@ -109,7 +109,7 @@ extension ItemRecord {
func setPropertyValue(
name: String,
value: PropertyDatabaseValue?,
addLog: Bool = true,
addLog: Bool = false,
db dbController: DatabaseController = AppController.shared.databaseController
) throws {
try dbController.writeSync { (db) in
......
......@@ -129,8 +129,17 @@ extension ItemRecord {
let databaseValue = try PropertyDatabaseValue(value: propertyValue, propertyType: expectedType)
try createdItem.setPropertyValue(name: unescapedPropertyName, value: databaseValue)
// If the item has file and it does not exist on disk, mark the file to be downloaded
if createdItem.type == "File",
propertyName == "filename",
let fileName = decodableValue.value as? String,
!FileManager.default.fileExists(atPath: FileStorageController.getFileStorageURL().appendingPathComponent(fileName).path) {
createdItem.fileState = .needsDownload
try createdItem.save(db: db)
}
}
// Check if it has edge and update
let _ = try EdgeRecord.fromSyncEdgeDict(item: itemRowId, fromSyncItemDict: dict, schema: schema, db: db)
......@@ -253,5 +262,70 @@ extension ItemRecord {
.fetchOne(db)
})
}
static func fileItemRecordToUpload() throws -> (item: ItemRecord, fileName: String)? {
guard let pool = AppController.shared.databaseController.databasePool else {
throw StringError(description: "Database pool not available")
}
/// Select the items to sync, giving priority to those marked as `syncHasPriority`
guard let item = try (pool.read { db in
try? ItemRecord
.filter(Column(ItemRecord.Columns.type.rawValue) == "File"
&& Column(ItemRecord.Columns.fileState.rawValue) == FileState.needsUpload)
.fetchOne(db)
}) else {
return nil
}
guard case let .string(fileName) = item.propertyValue("filename") else {
return nil
}
return (item: item, fileName: fileName)
}
static func didUploadFileForItem(_ item: ItemRecord) throws {
guard let rowId = item.rowId,
let fetchedItem = ItemRecord.fetchWithID(rowId) else {
return
}
fetchedItem.fileState = .noChanges
try fetchedItem.save()
}
static func fileItemRecordToDownload() throws -> (item: ItemRecord, sha256: String, fileName: String)? {
guard let pool = AppController.shared.databaseController.databasePool else {
throw StringError(description: "Database pool not available")
}
/// Select the items to sync, giving priority to those marked as `syncHasPriority`
guard let item = try (pool.read { db in
try? ItemRecord
.filter(Column(ItemRecord.Columns.type.rawValue) == "File"
&& Column(ItemRecord.Columns.fileState.rawValue) == FileState.needsDownload)
.fetchOne(db)
}) else {
return nil
}
guard case let .string(sha256) = item.propertyValue("sha256"),
case let .string(fileName) = item.propertyValue("filename") else {
return nil
}
return (item: item, sha256: sha256, fileName: fileName)
}
static func didDownloadFileForItem(_ item: ItemRecord) throws {
guard let rowId = item.rowId,
let fetchedItem = ItemRecord.fetchWithID(rowId) else {
return
}
fetchedItem.fileState = .noChanges
try fetchedItem.save()
}
}
......@@ -15,6 +15,13 @@ enum SyncState: String, Codable, Hashable, DatabaseValueConvertible {
case failed
}
enum FileState: String, Codable, Hashable, DatabaseValueConvertible {
case skip
case needsUpload
case needsDownload
case noChanges
}
class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
static override var databaseTableName: String { "items" }
......@@ -27,6 +34,8 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
var deleted: Bool
var syncState: SyncState
var fileState: FileState
var syncHasPriority: Bool
static let edges = hasMany(EdgeRecord.self, using: EdgeRecord.ownerForeignKey)
......@@ -35,7 +44,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
static var realProperties = hasMany(RealRecord.self, using: RealRecord.itemForeignKey)
enum Columns: String, ColumnExpression {
case rowId, id, type, dateCreated, dateModified, dateServerModified, deleted, syncState, syncHasPriority
case rowId, id, type, dateCreated, dateModified, dateServerModified, deleted, syncState, fileState, syncHasPriority
}
static func == (lhs: ItemRecord, rhs: ItemRecord) -> Bool {
......@@ -148,7 +157,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
}
internal init(rowId: Int64? = nil, id: StringUUID = StringUUID(), type: String, dateCreated: Date? = nil, dateModified: Date? = nil, dateServerModified: Date? = nil, deleted: Bool = false, syncState: SyncState = .create, syncHasPriority: Bool = false) {
internal init(rowId: Int64? = nil, id: StringUUID = StringUUID(), type: String, dateCreated: Date? = nil, dateModified: Date? = nil, dateServerModified: Date? = nil, deleted: Bool = false, syncState: SyncState = .create, fileState: FileState = .skip, syncHasPriority: Bool = false) {
self.rowId = rowId
self.id = id
self.type = type
......@@ -157,6 +166,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
self.dateServerModified = dateServerModified ?? Date.distantPast
self.deleted = deleted
self.syncState = syncState
self.fileState = fileState
self.syncHasPriority = syncHasPriority
super.init()
}
......@@ -170,6 +180,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
self.dateServerModified = row[Columns.dateServerModified]
self.deleted = row[Columns.deleted]
self.syncState = row[Columns.syncState]
self.fileState = row[Columns.fileState]
self.syncHasPriority = row[Columns.syncHasPriority]
super.init(row: row)
}
......@@ -183,6 +194,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
container[Columns.dateServerModified] = dateServerModified
container[Columns.deleted] = deleted
container[Columns.syncState] = syncState
container[Columns.fileState] = fileState
container[Columns.syncHasPriority] = syncHasPriority
}
......@@ -204,6 +216,7 @@ class ItemRecord: BaseRecord, Equatable, Hashable, Identifiable, Codable {
// This is direct from the latest version so no need to sync again
self.syncState = .noChanges
self.fileState = .skip
self.syncHasPriority = false
super.init()
}
......
......@@ -27,39 +27,18 @@ class FileStorageController {
}
}
#if targetEnvironment(simulator)
// On simulator, use a local folder for easy inspection
guard let homeDir = ProcessInfo.processInfo.environment["SIMULATOR_HOST_HOME"]
else { fatalError("Couldn't find home directory from simulator environment") }
let memriFileURL = URL(fileURLWithPath: homeDir, isDirectory: true)
.appendingPathComponent(
"memriDevData/fileStore",
isDirectory: true
)
createIfDoesntExist(directoryURL: memriFileURL)
return memriFileURL
#else
// On device, store under documents
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let documentsDirectory = paths[0]
let memriFileURL = documentsDirectory.appendingPathComponent(
"fileStore",
isDirectory: true
)
createIfDoesntExist(directoryURL: memriFileURL)
return memriFileURL
#endif
// On device, store under documents
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
let documentsDirectory = paths[0]
let memriFileURL = documentsDirectory.appendingPathComponent(
"fileStore",
isDirectory: true
)
createIfDoesntExist(directoryURL: memriFileURL)
return memriFileURL
}
static func getURLForFile(withUUID uuid: String) -> URL {
// Little hack to make our demo data work
let split = uuid.split(separator: ".")
let fileExt = split.count > 1 ? String(split.last ?? "jpg") : "jpg"
if let fileName = split.first, let url = Bundle.main.url(forResource: "demoAssets/\(fileName)", withExtension: fileExt) {
return url
}
// End hack
return getFileStorageURL().appendingPathComponent(uuid, isDirectory: false)
}
......
......@@ -17,6 +17,8 @@ enum SyncControllerState: String, Codable, Hashable, DatabaseValueConvertible {
case uploadedSchemaEdges
case uploadedItems
case uploadedEdges
case uploadedFiles
case downloadedFiles
case done
case failed
}
......@@ -43,6 +45,10 @@ class SyncController {
case .uploadedItems:
try? uploadEdges()
case .uploadedEdges:
try? uploadFiles()
case .uploadedFiles:
try? downloadFiles()
case .downloadedFiles:
finishSync()
case .failed:
finishSync()
......@@ -200,6 +206,46 @@ class SyncController {
})
}
private func uploadFiles() throws {
guard let (item: item, fileName: fileName) = try ItemRecord.fileItemRecordToUpload() else {
self.state = .uploadedFiles
return
}
print("Uploading File: ", fileName)
try uploadFile(uuid: fileName) {[weak self] data, error in
guard error == nil else {
self?.lastError = error
self?.state = .failed
return
}
try? ItemRecord.didUploadFileForItem(item)
try? self?.uploadFiles()
}
}
private func downloadFiles() throws {
guard let (item: item, sha256: sha256, fileName: fileName) = try ItemRecord.fileItemRecordToDownload() else {
self.state = .downloadedFiles
return
}
print("Downloading File: ", fileName)
try downloadFile(sha256: sha256, fileName: fileName, completion: { [weak self] error in
guard error == nil else {
self?.lastError = error
self?.state = .failed
return
}
try? ItemRecord.didDownloadFileForItem(item)
try? self?.downloadFiles()
})
}
private func finishSync() {
self.state = .done
self.syncing = false
......@@ -207,6 +253,7 @@ class SyncController {
self.state = .idle
self.completion = nil
print("Sync Complete!")
}
func makeSyncSchemaPropertiesData() throws -> PodAPIPayload.BulkAction {
......@@ -306,4 +353,41 @@ class SyncController {
completion?(result.data, result.error)
} .store(in: &subscriptions)
}
public func uploadFile(uuid: String, completion: ((Data?, Error?) -> Void)?) throws {
guard let connectionConfig = AppController.shared.podConnectionConfig
else { throw StringError(description: "No pod connection config") }
let fileURL = FileStorageController.getURLForFile(withUUID: uuid)
let request = try PodAPIUploadRequest.uploadFile(fileURL: fileURL, connectionConfig: connectionConfig)
let networkCall = try request.execute(connectionConfig: connectionConfig)
networkCall.sink { (result) in
if result.error != nil, let resultData = result.data {
let errorString = String(data: resultData, encoding: .utf8)
if errorString == "Failure: File already exists" {
completion?(result.data, nil)
return
}
print("ERROR: ", errorString ?? "")
}
completion?(result.data, result.error)
} .store(in: &subscriptions)
}
public func downloadFile(sha256: String, fileName: String, completion: ((Error?) -> Void)?) throws {
guard let connectionConfig = AppController.shared.podConnectionConfig
else { throw StringError(description: "No pod connection config") }
let request = PodAPIDownloadRequest.downloadFile(fileSHAHash: sha256, fileUID: fileName)
let networkCall = try request.execute(connectionConfig: connectionConfig)
networkCall.sink { (result) in
if result.error != nil {
print("ERROR: ", result.error?.localizedDescription ?? "")
}
completion?(result.error)
} .store(in: &subscriptions)
}
}
......@@ -88,3 +88,16 @@ extension RangeReplaceableCollection where Element == Character {
return first.uppercased() + dropFirst()
}
}
extension String {
var fileName: String? {
let name = URL(fileURLWithPath: self).deletingPathExtension().lastPathComponent
return name.isEmpty ? nil : name
}
var fileExtension: String? {
let ext = URL(fileURLWithPath: self).pathExtension
return ext.isEmpty ? nil : ext
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment