diff --git a/backend/toolshed/api/offlinedata.py b/backend/toolshed/api/offlinedata.py index cb64ae1..1d25547 100644 --- a/backend/toolshed/api/offlinedata.py +++ b/backend/toolshed/api/offlinedata.py @@ -5,18 +5,40 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from authentication.signature_auth import SignatureAuthentication +from toolshed.offlinedata import ( + inventory_rows, friend_rows, location_rows, inventory_files, rows_to_csv, + import_locations, import_friends, import_inventory, +) -def user_data(): +def local_user_or_none(identity): + """Resolve the local ToolshedUser associated with an authenticated KnownIdentity, if any. + + Returns None if the identity belongs to an external/remote user with no local account. + """ + if identity is None: + return None + return identity.user.first() + + +def user_data(user): import io import zipfile zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file: - for file_name, data in [('1.txt', io.BytesIO(b'111')), - ('2.txt', io.BytesIO(b'222'))]: - zip_file.writestr(file_name, data.getvalue()) + inventory_csv = b''.join(rows_to_csv(inventory_rows(user))) + zip_file.writestr('inventory.csv', inventory_csv) + + friends_csv = b''.join(rows_to_csv(friend_rows(user))) + zip_file.writestr('friends.csv', friends_csv) + + locations_csv = b''.join(rows_to_csv(location_rows(user))) + zip_file.writestr('locations.csv', locations_csv) + + for arcname, data in inventory_files(user): + zip_file.writestr(arcname, data) return zip_buffer.getvalue() @@ -30,30 +52,118 @@ def parse_user_data(data): yield file_name, zip_file.read(file_name) +def import_files(zip_file): + """Fault-tolerant extraction of the 'files/' subfolder into File objects. + + Returns a dict mapping the zip arcname to the created/existing File instance. Entries that + fail to read or save are silently skipped so a single corrupt attachment doesn't abort the import. + """ + from hashlib import sha256 + import mimetypes + + from django.core.files.base import ContentFile + from files.models import File + + result = {} + for name in zip_file.namelist(): + if not name.startswith('files/') or name == 'files/': + continue + try: + data = zip_file.read(name) + content_hash = sha256(data).hexdigest() + mime_type, _ = mimetypes.guess_type(name) + file_obj = File.objects.filter(hash=content_hash).first() + if file_obj is None: + file_obj = File.objects.create( + file=ContentFile(data, content_hash), + mime_type=mime_type or 'application/octet-stream', + hash=content_hash, + ) + result[name] = file_obj + except Exception as error: + print(f'Skipping file "{name}" during import: {error}') + return result + + + +def import_user_data(user, data): + """Fault-tolerant import of an export zip produced by `user_data()`. + + Any of 'inventory.csv', 'friends.csv', 'locations.csv' or the 'files/' subfolder may be + missing; whatever is present is imported and everything else is silently skipped. + """ + import io + import zipfile + + summary = {'locations': 0, 'friends': 0, 'inventory_items': 0, 'files': 0} + + with zipfile.ZipFile(io.BytesIO(data), 'r') as zip_file: + names = set(zip_file.namelist()) + + available_files = import_files(zip_file) + summary['files'] = len(available_files) + + if 'locations.csv' in names: + summary['locations'] = import_locations(user, zip_file.read('locations.csv')) + + if 'friends.csv' in names: + summary['friends'] = import_friends(user, zip_file.read('friends.csv')) + + if 'inventory.csv' in names: + summary['inventory_items'] = import_inventory(user, zip_file.read('inventory.csv'), available_files) + + return summary + + +def _extract_zip_bytes(zip_payload): + """Normalize the incoming 'zip' request payload (uploaded file, base64 string, or raw bytes).""" + import base64 + + if hasattr(zip_payload, 'read'): + return zip_payload.read() + if isinstance(zip_payload, str): + try: + return base64.b64decode(zip_payload, validate=True) + except Exception: + return zip_payload.encode('utf-8') + return zip_payload + + @api_view(['POST']) @permission_classes([IsAuthenticated]) @authentication_classes([SignatureAuthentication]) def import_data(request, format=None): - zip = request.data.get('zip') - if not zip: + local_user = local_user_or_none(request.user) + if local_user is None: + return Response({'detail': 'This endpoint is only available to local users'}, status=403) + zip_payload = request.data.get('zip') + if not zip_payload: return Response(status=400) - for file_name, data in parse_user_data(zip): - print(file_name, data) - return Response(status=200) + try: + zip_bytes = _extract_zip_bytes(zip_payload) + summary = import_user_data(local_user, zip_bytes) + except Exception as error: + return Response({'detail': f'Could not read zip file: {error}'}, status=400) + return Response(summary, status=200) @api_view(['GET']) @permission_classes([IsAuthenticated]) @authentication_classes([SignatureAuthentication]) def export_data(request, format=None): - return HttpResponse(user_data(), content_type='application/zip', status=200) + local_user = local_user_or_none(request.user) + if local_user is None: + return Response({'detail': 'This endpoint is only available to local users'}, status=403) + return HttpResponse(user_data(local_user), content_type='application/zip', status=200) @api_view(['DELETE']) @permission_classes([IsAuthenticated]) @authentication_classes([SignatureAuthentication]) def delete_account(request, format=None): - pass + local_user = local_user_or_none(request.user) + if local_user is None: + return Response({'detail': 'This endpoint is only available to local users'}, status=403) urlpatterns = [ diff --git a/backend/toolshed/offlinedata.py b/backend/toolshed/offlinedata.py new file mode 100644 index 0000000..98de35e --- /dev/null +++ b/backend/toolshed/offlinedata.py @@ -0,0 +1,312 @@ +"""Data helpers for building/importing a user's offline export. + +These functions deal with the toolshed models and CSV row shapes only - they know nothing +about the zip container format or how the request is authenticated. See toolshed/api/offlinedata.py +for the zip-building/parsing and the API views themselves. +""" + + +def inventory_rows(user): + """Generator that yields the given user's inventory items as flattened dicts, one per row.""" + import mimetypes + + from toolshed.models import InventoryItem + + items = (InventoryItem.objects + .filter(owner=user) + .select_related('category', 'storage_location') + .prefetch_related('tags', 'itemproperty_set__property', 'files')) + + for item in items: + file_paths = [] + for f in item.files.all(): + extension = mimetypes.guess_extension(f.mime_type) or '' + file_paths.append(f'files/{f.hash}{extension}') + + yield { + 'id': item.id, + 'name': item.name or '', + 'description': item.description or '', + 'category': str(item.category) if item.category else '', + 'availability_policy': item.availability_policy, + 'owned_quantity': item.owned_quantity, + 'storage_location': str(item.storage_location) if item.storage_location else '', + 'tags': ', '.join(tag.name for tag in item.tags.all()), + 'properties': ', '.join(f"{ip.property.name}={ip.value}" for ip in item.itemproperty_set.all()), + 'files': ', '.join(file_paths), + 'created_at': item.created_at.isoformat() if item.created_at else '', + } + + +def friend_rows(user): + """Generator that yields the given user's friends (known identities) as flattened dicts, one per row.""" + friends = user.friends.all() + + for friend in friends: + yield { + 'username': friend.username, + 'domain': friend.domain, + 'handle': f'{friend.username}@{friend.domain}', + 'public_key': friend.public_key, + } + + +def location_path(location): + """Recursively build the '/'-joined path of a StorageLocation, following its parent chain.""" + if location.parent: + return location_path(location.parent) + '/' + location.name + return location.name + + +def location_rows(user): + """Generator that yields the given user's storage locations as flattened dicts, one per row.""" + from toolshed.models import StorageLocation + + locations = StorageLocation.objects.filter(owner=user).select_related('category', 'parent') + + for location in locations: + yield { + 'id': location.id, + 'name': location.name, + 'description': location.description or '', + 'category': str(location.category) if location.category else '', + 'parent': str(location.parent) if location.parent else '', + 'path': location_path(location), + } + + +def inventory_files(user): + """Generator that yields (arcname, data) for each unique file attached to the user's inventory items. + + Files are deduplicated by hash and placed under a 'files/' subfolder, keeping their original + extension (guessed from mime_type) so attachments and images remain viewable once extracted. + """ + import mimetypes + + from toolshed.models import InventoryItem + + seen_hashes = set() + items = InventoryItem.objects.filter(owner=user).prefetch_related('files') + + for item in items: + for file in item.files.all(): + if file.hash in seen_hashes: + continue + seen_hashes.add(file.hash) + + extension = mimetypes.guess_extension(file.mime_type) or '' + arcname = f'files/{file.hash}{extension}' + + file.file.open('rb') + try: + data = file.file.read() + finally: + file.file.close() + + yield arcname, data + + +def rows_to_csv(rows, fieldnames=None, encoding='utf-8'): + """Generator that consumes an iterable of dicts and yields encoded CSV data chunk by chunk.""" + import csv + + class _Echo: + """A file-like object whose write() just returns what was passed, for streaming csv.writer output.""" + + def write(self, value): + return value + + rows = iter(rows) + try: + first_row = next(rows) + except StopIteration: + return + + if fieldnames is None: + fieldnames = list(first_row.keys()) + + writer = csv.DictWriter(_Echo(), fieldnames=fieldnames) + yield writer.writeheader().encode(encoding) + yield writer.writerow(first_row).encode(encoding) + for row in rows: + yield writer.writerow(row).encode(encoding) + + +def _read_csv_rows(data, encoding='utf-8'): + """Decode CSV bytes and yield rows as dicts keyed by the header labels (not column offsets).""" + import csv + import io + + reader = csv.DictReader(io.StringIO(data.decode(encoding))) + for row in reader: + yield row + + +def _field_is_optional(model, field_name): + """Return True if a field may be omitted: it's a relation, has a default, or allows null/blank.""" + try: + field = model._meta.get_field(field_name) + except Exception: + return True + if getattr(field, 'many_to_many', False) or getattr(field, 'one_to_many', False): + return True + return bool(getattr(field, 'null', False) or getattr(field, 'blank', False) or field.has_default()) + + +def get_or_create_category(path): + """Resolve or create a Category from a '/'-separated path such as 'Electronics/Cables'.""" + from toolshed.models import Category + + parent = None + category = None + for part in (p for p in path.split('/') if p): + category, _ = Category.objects.get_or_create(name=part, parent=parent, defaults={'origin': 'import'}) + parent = category + return category + + +def import_locations(user, data): + """Fault-tolerant import of locations.csv into StorageLocations owned by `user`. + + Rows are read by header label. Rows missing the required 'name' column are skipped. + Optional columns ('description', 'category') are simply omitted if absent. Locations are + processed in path-depth order so a child's parent already exists by the time it's needed. + """ + from toolshed.models import StorageLocation + + rows = list(_read_csv_rows(data)) + rows.sort(key=lambda row: (row.get('path') or row.get('name') or '').count('/')) + + resolved_by_path = {} + imported = 0 + + for row in rows: + try: + name = (row.get('name') or '').strip() + if not name and not _field_is_optional(StorageLocation, 'name'): + continue # required column missing, skip this row + + path = (row.get('path') or name).strip() + parent = None + if '/' in path: + parent = resolved_by_path.get(path.rsplit('/', 1)[0]) + + category = None + category_path = row.get('category') + if category_path: + category = get_or_create_category(category_path) + + location, _ = StorageLocation.objects.update_or_create( + owner=user, name=name, parent=parent, + defaults={ + 'description': row.get('description', '') or '', + 'category': category, + }, + ) + resolved_by_path[path] = location + imported += 1 + except Exception as error: + print(f'Skipping location row {row}: {error}') + + return imported + + +def import_friends(user, data): + """Fault-tolerant import of friends.csv, adding valid entries to the user's known friends.""" + from authentication.models import KnownIdentity + + imported = 0 + for row in _read_csv_rows(data): + try: + username = row.get('username') + domain = row.get('domain') + handle = row.get('handle') + if (not username or not domain) and handle and '@' in handle: + username, domain = handle.split('@', 1) + + public_key = row.get('public_key') + if not username or not domain or not public_key: + continue # required fields missing, skip this row + + identity, _ = KnownIdentity.objects.get_or_create( + username=username, domain=domain, + defaults={'public_key': public_key}, + ) + user.public_identity.friends.add(identity) + imported += 1 + except Exception as error: + print(f'Skipping friend row {row}: {error}') + + return imported + + +def import_inventory(user, data, available_files): + """Fault-tolerant import of inventory.csv into InventoryItems owned by `user`. + + `available_files` maps the 'files' column's paths to File instances successfully extracted + from the zip; references to missing files are ignored rather than failing the row. + """ + from toolshed.models import InventoryItem, ItemProperty, StorageLocation, Tag, Property + + imported = 0 + for row in _read_csv_rows(data): + try: + name = (row.get('name') or '').strip() + + file_paths = [p.strip() for p in (row.get('files') or '').split(',') if p.strip()] + files = [available_files[p] for p in file_paths if p in available_files] + + if not name and not files: + continue # InventoryItem requires a name or at least one file + + category = None + category_path = row.get('category') + if category_path: + category = get_or_create_category(category_path) + + storage_location = None + location_path_value = row.get('storage_location') + if location_path_value: + storage_location = StorageLocation.objects.filter( + owner=user, name=location_path_value.rsplit('/', 1)[-1]).first() + + try: + owned_quantity = int(row.get('owned_quantity') or 1) + except (TypeError, ValueError): + owned_quantity = 1 + + item = InventoryItem.objects.create( + owner=user, + name=name or None, + description=row.get('description', '') or '', + category=category, + availability_policy=row.get('availability_policy') or 'private', + owned_quantity=owned_quantity, + storage_location=storage_location, + ) + + for tag_name in (row.get('tags') or '').split(','): + tag_name = tag_name.strip() + if tag_name: + tag, _ = Tag.objects.get_or_create(name=tag_name, category=None, + defaults={'origin': 'import'}) + item.tags.add(tag, through_defaults={}) + + for prop_entry in (row.get('properties') or '').split(','): + prop_entry = prop_entry.strip() + if '=' not in prop_entry: + continue + prop_name, value = prop_entry.split('=', 1) + prop, _ = Property.objects.get_or_create(name=prop_name.strip(), category=None, + defaults={'origin': 'import'}) + ItemProperty.objects.create(inventory_item=item, property=prop, value=value.strip()) + + for file in files: + item.files.add(file) + + imported += 1 + except Exception as error: + print(f'Skipping inventory row {row}: {error}') + + return imported + diff --git a/frontend/src/views/settings/Data.vue b/frontend/src/views/settings/Data.vue index f8e2268..2aa3365 100644 --- a/frontend/src/views/settings/Data.vue +++ b/frontend/src/views/settings/Data.vue @@ -103,20 +103,50 @@ export default { exportKey() { const key = this.userIdentityRecord; }, + fileToBase64(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const buffer = reader.result; + if (!(buffer instanceof ArrayBuffer)) { + reject(new Error('Could not read file')); + return; + } + const data = new Uint8Array(buffer); + const base64 = btoa(data.reduce((acc, byte) => acc + String.fromCharCode(byte), '')); + resolve(base64); + }; + reader.onerror = (error) => reject(error); + reader.readAsArrayBuffer(file); + }); + }, async importData() { if (!this.selectedFile) { alert('Please select a file to import'); return; } - alert('Data import not implemented'); - return; - - const formData = new FormData(); - formData.append('file', this.selectedFile); - console.log(formData); - const servers = await this.getHomeServers(); - const data = await servers.postRaw(this.signHashedAuth, '/api/import/', formData); - console.log(data); + try { + const base64 = await this.fileToBase64(this.selectedFile); + const servers = await this.getHomeServers(); + const summary = await servers.post(this.signAuth, '/api/import/', {zip: base64}); + if (summary && summary.detail) { + alert('Data import failed: ' + summary.detail); + return; + } + alert('Data imported successfully: ' + + `${summary.inventory_items || 0} inventory items, ` + + `${summary.friends || 0} friends, ` + + `${summary.locations || 0} locations, ` + + `${summary.files || 0} files.`); + this.selectedFile = null; + const fileInput = document.getElementById('inputFile'); + if (fileInput) { + fileInput.value = ''; + } + } catch (error) { + console.error('Data import failed', error); + alert('Data import failed: ' + error); + } }, async exportData() { const servers = await this.getHomeServers(); diff --git a/testdata/user-a.key b/testdata/user-a.key new file mode 100644 index 0000000..013ea42 --- /dev/null +++ b/testdata/user-a.key @@ -0,0 +1 @@ +f0cd0b223efb7ca24cd8ad6e39471fb1b4f69a20a3e5c6fdb41c9985b7dbf381 \ No newline at end of file diff --git a/testdata/user-a.zip b/testdata/user-a.zip new file mode 100644 index 0000000..c49f588 Binary files /dev/null and b/testdata/user-a.zip differ diff --git a/testdata/user-b.key b/testdata/user-b.key new file mode 100644 index 0000000..7bc5e63 --- /dev/null +++ b/testdata/user-b.key @@ -0,0 +1 @@ +6b186fde49d12b7108e5246b5279be660fba185cfd514cb0d949fea5d7db9131 \ No newline at end of file diff --git a/testdata/user-b.zip b/testdata/user-b.zip new file mode 100644 index 0000000..cccccc4 Binary files /dev/null and b/testdata/user-b.zip differ