From 0a73acc35cfbcdeba21114b734d5b59044fd642b Mon Sep 17 00:00:00 2001 From: Nutcake Date: Tue, 9 May 2023 10:52:13 +0200 Subject: [PATCH] Lay groundwork for asset uploads --- lib/apis/asset_api.dart | 107 ++++++++++++++++++ lib/models/asset/asset_diff.dart | 34 ++++++ lib/models/asset/asset_upload_data.dart | 46 ++++++++ lib/models/asset/neos_db_asset.dart | 26 +++++ lib/models/asset/preprocess_status.dart | 41 +++++++ lib/models/asset/record.dart | 142 ++++++++++++++++++++++++ 6 files changed, 396 insertions(+) create mode 100644 lib/apis/asset_api.dart create mode 100644 lib/models/asset/asset_diff.dart create mode 100644 lib/models/asset/asset_upload_data.dart create mode 100644 lib/models/asset/neos_db_asset.dart create mode 100644 lib/models/asset/preprocess_status.dart create mode 100644 lib/models/asset/record.dart diff --git a/lib/apis/asset_api.dart b/lib/apis/asset_api.dart new file mode 100644 index 0000000..31c6efd --- /dev/null +++ b/lib/apis/asset_api.dart @@ -0,0 +1,107 @@ + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; +import 'package:contacts_plus_plus/models/message.dart'; +import 'package:http/http.dart' as http; + +import 'package:contacts_plus_plus/clients/api_client.dart'; +import 'package:contacts_plus_plus/models/asset/asset_upload_data.dart'; +import 'package:contacts_plus_plus/models/asset/neos_db_asset.dart'; +import 'package:contacts_plus_plus/models/asset/preprocess_status.dart'; +import 'package:contacts_plus_plus/models/asset/record.dart'; +import 'package:path/path.dart'; + +class AssetApi { + static Future preprocessRecord(ApiClient client, {required Record record}) async { + final response = await client.post( + "/users/${record.ownerId}/records/${record.id}/preprocess", body: jsonEncode(record.toMap())); + ApiClient.checkResponse(response); + final body = jsonDecode(response.body); + return PreprocessStatus.fromMap(body); + } + + static Future getPreprocessStatus(ApiClient client, + {required PreprocessStatus preprocessStatus}) async { + final response = await client.get( + "/users/${preprocessStatus.ownerId}/records/${preprocessStatus.recordId}/preprocess/${preprocessStatus.id}" + ); + ApiClient.checkResponse(response); + final body = jsonDecode(response.body); + return PreprocessStatus.fromMap(body); + } + + static Future beginUploadAsset(ApiClient client, {required NeosDBAsset asset}) async { + final response = await client.post("/users/${client.userId}/assets/${asset.hash}/chunks?bytes=${asset.bytes}"); + ApiClient.checkResponse(response); + final body = jsonDecode(response.body); + final res = AssetUploadData.fromMap(body); + if (res.uploadState == UploadState.failed) throw body; + return res; + } + + static Future upsertRecord(ApiClient client, {required Record record}) async { + final body = jsonEncode(record.toMap()); + final response = await client.put("/users/${client.userId}/records/${record.id}", body: body); + ApiClient.checkResponse(response); + } + + static Future uploadAssets(ApiClient client, {required NeosDBAsset asset, required Uint8List data}) async { + final request = http.MultipartRequest( + "POST", + ApiClient.buildFullUri("/users/${client.userId}/assets/${asset.hash}"), + ) + ..files.add(http.MultipartFile.fromBytes("file", data)); + final response = await request.send(); + final body = jsonDecode(await response.stream.bytesToString()); + return body; + } + + static Future uploadFile(ApiClient client, {required File file, required String machineId}) async { + final data = await file.readAsBytes(); + final asset = NeosDBAsset.fromData(data); + final assetUri = "local://$machineId/${asset.hash}${extension(file.path)}"; + final record = Record( + id: Record.generateId(), + assetUri: assetUri, + name: basenameWithoutExtension(file.path), + tags: [ + "message_item", + "message_id:${Message.generateId()}" + ], + recordType: "texture", + thumbnailUri: assetUri, + isPublic: false, + isForPatreons: false, + isListed: false, + isDeleted: false, + neosDBManifest: [ + asset, + ], + localVersion: 1, + lastModifyingUserId: client.userId, + lastModifyingMachineId: machineId, + lastModificationTime: DateTime.now().toUtc(), + creationTime: DateTime.now().toUtc(), + ownerId: client.userId, + ); + + var status = await preprocessRecord(client, record: record); + while (status.state == RecordPreprocessState.preprocessing) { + await Future.delayed(const Duration(seconds: 1)); + status = await getPreprocessStatus(client, preprocessStatus: status); + } + + if (status.state != RecordPreprocessState.success) { + throw "Record Preprocessing failed: ${status.failReason}"; + } + + final uploadData = await beginUploadAsset(client, asset: asset); + if (uploadData.uploadState == UploadState.failed) { + throw "Asset upload failed: ${uploadData.uploadState.name}"; + } + + await uploadAssets(client, asset: asset, data: data); + return record; + } +} \ No newline at end of file diff --git a/lib/models/asset/asset_diff.dart b/lib/models/asset/asset_diff.dart new file mode 100644 index 0000000..cd97d30 --- /dev/null +++ b/lib/models/asset/asset_diff.dart @@ -0,0 +1,34 @@ + +class AssetDiff { + final String hash; + final int bytes; + final Diff state; + final bool isUploaded; + + const AssetDiff({required this.hash, required this.bytes, required this.state, required this.isUploaded}); + + factory AssetDiff.fromMap(Map map) { + return AssetDiff( + hash: map["hash"], + bytes: map["bytes"], + state: Diff.fromInt(map["state"]), + isUploaded: map["isUploaded"], + ); + } +} + +enum Diff { + added, + unchanged, + removed; + + factory Diff.fromInt(int? idx) { + return Diff.values[idx ?? 1]; + } + + factory Diff.fromString(String? text) { + return Diff.values.firstWhere((element) => element.name.toLowerCase() == text?.toLowerCase(), + orElse: () => Diff.unchanged, + ); + } +} \ No newline at end of file diff --git a/lib/models/asset/asset_upload_data.dart b/lib/models/asset/asset_upload_data.dart new file mode 100644 index 0000000..6df0555 --- /dev/null +++ b/lib/models/asset/asset_upload_data.dart @@ -0,0 +1,46 @@ + +enum UploadState { + uploadingChunks, + finalizing, + uploaded, + failed, + unknown; + + factory UploadState.fromString(String? text) { + return UploadState.values.firstWhere((element) => element.name.toLowerCase() == text?.toLowerCase(), + orElse: () => UploadState.unknown, + ); + } +} + +class AssetUploadData { + final String signature; + final String variant; + final String ownerId; + final int totalBytes; + final int chunkSize; + final int totalChunks; + final UploadState uploadState; + + const AssetUploadData({ + required this.signature, + required this.variant, + required this.ownerId, + required this.totalBytes, + required this.chunkSize, + required this.totalChunks, + required this.uploadState, + }); + + factory AssetUploadData.fromMap(Map map) { + return AssetUploadData( + signature: map["signature"], + variant: map["variant"] ?? "", + ownerId: map["ownerId"] ?? "", + totalBytes: map["totalBytes"] ?? -1, + chunkSize: map["chunkSize"] ?? -1, + totalChunks: map["totalChunks"] ?? -1, + uploadState: UploadState.fromString(map["uploadStat"]), + ); + } +} \ No newline at end of file diff --git a/lib/models/asset/neos_db_asset.dart b/lib/models/asset/neos_db_asset.dart new file mode 100644 index 0000000..8b0c64e --- /dev/null +++ b/lib/models/asset/neos_db_asset.dart @@ -0,0 +1,26 @@ +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; + +class NeosDBAsset { + final String hash; + final int bytes; + + const NeosDBAsset({required this.hash, required this.bytes}); + + factory NeosDBAsset.fromMap(Map map) { + return NeosDBAsset(hash: map["hash"] ?? "", bytes: map["bytes"] ?? -1); + } + + factory NeosDBAsset.fromData(Uint8List data) { + final digest = sha256.convert(data); + return NeosDBAsset(hash: digest.toString().replaceAll("-", "").toLowerCase(), bytes: data.length); + } + + Map toMap() { + return { + "hash": hash, + "bytes": bytes, + }; + } +} \ No newline at end of file diff --git a/lib/models/asset/preprocess_status.dart b/lib/models/asset/preprocess_status.dart new file mode 100644 index 0000000..e6e8e8d --- /dev/null +++ b/lib/models/asset/preprocess_status.dart @@ -0,0 +1,41 @@ +import 'package:contacts_plus_plus/models/asset/asset_diff.dart'; + +enum RecordPreprocessState +{ + preprocessing, + success, + failed; + + factory RecordPreprocessState.fromString(String? text) { + return RecordPreprocessState.values.firstWhere((element) => element.name.toLowerCase() == text?.toLowerCase(), + orElse: () => RecordPreprocessState.failed, + ); + } +} + + +class PreprocessStatus { + final String id; + final String ownerId; + final String recordId; + final RecordPreprocessState state; + final num progress; + final String failReason; + final List resultDiffs; + + const PreprocessStatus({required this.id, required this.ownerId, required this.recordId, required this.state, + required this.progress, required this.failReason, required this.resultDiffs, + }); + + factory PreprocessStatus.fromMap(Map map) { + return PreprocessStatus( + id: map["id"], + ownerId: map["ownerId"], + recordId: map["recordId"], + state: RecordPreprocessState.fromString(map["state"]), + progress: map["progress"], + failReason: map["failReason"] ?? "", + resultDiffs: (map["resultDiffs"] as List? ?? []).map((e) => AssetDiff.fromMap(e)).toList(), + ); + } +} \ No newline at end of file diff --git a/lib/models/asset/record.dart b/lib/models/asset/record.dart new file mode 100644 index 0000000..5e9a0a2 --- /dev/null +++ b/lib/models/asset/record.dart @@ -0,0 +1,142 @@ +import 'package:contacts_plus_plus/models/asset/neos_db_asset.dart'; +import 'package:uuid/uuid.dart'; + +class Record { + final String id; + final String ownerId; + final String? assetUri; + final int globalVersion; + final int localVersion; + final String name; + final String? description; + final List? tags; + final String recordType; + final String? thumbnailUri; + final bool isPublic; + final bool isForPatreons; + final bool isListed; + final bool isDeleted; + final DateTime? lastModificationTime; + final List neosDBManifest; + final String lastModifyingUserId; + final String lastModifyingMachineId; + final DateTime? creationTime; + + const Record({ + required this.id, + required this.ownerId, + this.assetUri, + this.globalVersion=0, + this.localVersion=0, + required this.name, + this.description, + this.tags, + required this.recordType, + this.thumbnailUri, + required this.isPublic, + required this.isListed, + required this.isDeleted, + required this.isForPatreons, + this.lastModificationTime, + required this.neosDBManifest, + required this.lastModifyingUserId, + required this.lastModifyingMachineId, + this.creationTime, + }); + + factory Record.fromMap(Map map) { + return Record( + id: map["id"], + ownerId: map["ownerId"], + assetUri: map["assetUri"], + globalVersion: map["globalVersion"] ?? 0, + localVersion: map["localVersion"] ?? 0, + name: map["name"] ?? "", + description: map["description"], + tags: (map["tags"] as List).map((e) => e.toString()).toList(), + recordType: map["recordType"] ?? "", + thumbnailUri: map["thumbnailUri"], + isPublic: map["isPublic"] ?? false, + isForPatreons: map["isForPatreons"] ?? false, + isListed: map["isListed"] ?? false, + isDeleted: map["isDeleted"] ?? false, + lastModificationTime: DateTime.tryParse(map["lastModificationTime"]), + neosDBManifest: (map["neosDBManifest"] as List? ?? []).map((e) => NeosDBAsset.fromMap(e)).toList(), + lastModifyingUserId: map["lastModifyingUserId"] ?? "", + lastModifyingMachineId: map["lastModifyingMachineId"] ?? "", + creationTime: DateTime.tryParse(map["lastModificationTime"]), + ); + } + + Record copyWith({ + String? id, + String? ownerId, + String? assetUri, + int? globalVersion, + int? localVersion, + String? name, + String? description, + List? tags, + String? recordType, + String? thumbnailUri, + bool? isPublic, + bool? isForPatreons, + bool? isListed, + bool? isDeleted, + DateTime? lastModificationTime, + List? neosDBManifest, + String? lastModifyingUserId, + String? lastModifyingMachineId, + DateTime? creationTime, + }) { + return Record( + id: id ?? this.id, + ownerId: ownerId ?? this.ownerId, + assetUri: assetUri ?? this.assetUri, + globalVersion: globalVersion ?? this.globalVersion, + localVersion: localVersion ?? this.localVersion, + name: name ?? this.name, + description: description ?? this.description, + tags: tags ?? this.tags, + recordType: recordType ?? this.recordType, + thumbnailUri: thumbnailUri ?? this.thumbnailUri, + isPublic: isPublic ?? this.isPublic, + isForPatreons: isForPatreons ?? this.isForPatreons, + isListed: isListed ?? this.isListed, + isDeleted: isDeleted ?? this.isDeleted, + lastModificationTime: lastModificationTime ?? this.lastModificationTime, + neosDBManifest: neosDBManifest ?? this.neosDBManifest, + lastModifyingUserId: lastModifyingUserId ?? this.lastModifyingUserId, + lastModifyingMachineId: lastModifyingMachineId ?? this.lastModifyingMachineId, + creationTime: creationTime ?? this.creationTime, + ); + } + + Map toMap() { + return { + "id": id, + "ownerId": ownerId, + "assetUri": assetUri, + "globalVersion": globalVersion, + "localVersion": localVersion, + "name": name, + "description": description, + "tags": tags, + "recordType": recordType, + "thumbnailUri": thumbnailUri, + "isPublic": isPublic, + "isForPatreons": isForPatreons, + "isListed": isListed, + "isDeleted": isDeleted, + "lastModificationTime": lastModificationTime?.toIso8601String(), + "neosDBManifest": neosDBManifest.map((e) => e.toMap()).toList(), + "lastModifyingUserId": lastModifyingUserId, + "lastModifyingMachineId": lastModifyingMachineId, + "creationTime": creationTime?.toIso8601String(), + }; + } + + static String generateId() { + return "R-${const Uuid().v4()}"; + } +} \ No newline at end of file