offline data prototype

backend/toolshed/api/offlinedata.py

@@ -5,18 +5,40 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 
 from authentication.signature_auth import SignatureAuthentication
+from toolshed.offlinedata import (
+    inventory_rows, friend_rows, location_rows, inventory_files, rows_to_csv,
+    import_locations, import_friends, import_inventory,
+)
 
 
-def user_data():
+def local_user_or_none(identity):
+    """Resolve the local ToolshedUser associated with an authenticated KnownIdentity, if any.
+
+    Returns None if the identity belongs to an external/remote user with no local account.
+    """
+    if identity is None:
+        return None
+    return identity.user.first()
+
+
+def user_data(user):
     import io
     import zipfile
 
     zip_buffer = io.BytesIO()
 
     with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file:
-        for file_name, data in [('1.txt', io.BytesIO(b'111')),
+        inventory_csv = b''.join(rows_to_csv(inventory_rows(user)))
-                                ('2.txt', io.BytesIO(b'222'))]:
+        zip_file.writestr('inventory.csv', inventory_csv)
-            zip_file.writestr(file_name, data.getvalue())
+
+        friends_csv = b''.join(rows_to_csv(friend_rows(user)))
+        zip_file.writestr('friends.csv', friends_csv)
+
+        locations_csv = b''.join(rows_to_csv(location_rows(user)))
+        zip_file.writestr('locations.csv', locations_csv)
+
+        for arcname, data in inventory_files(user):
+            zip_file.writestr(arcname, data)
 
     return zip_buffer.getvalue()
 
@@ -30,30 +52,118 @@ def parse_user_data(data):
             yield file_name, zip_file.read(file_name)
 
 
+def import_files(zip_file):
+    """Fault-tolerant extraction of the 'files/' subfolder into File objects.
+
+    Returns a dict mapping the zip arcname to the created/existing File instance. Entries that
+    fail to read or save are silently skipped so a single corrupt attachment doesn't abort the import.
+    """
+    from hashlib import sha256
+    import mimetypes
+
+    from django.core.files.base import ContentFile
+    from files.models import File
+
+    result = {}
+    for name in zip_file.namelist():
+        if not name.startswith('files/') or name == 'files/':
+            continue
+        try:
+            data = zip_file.read(name)
+            content_hash = sha256(data).hexdigest()
+            mime_type, _ = mimetypes.guess_type(name)
+            file_obj = File.objects.filter(hash=content_hash).first()
+            if file_obj is None:
+                file_obj = File.objects.create(
+                    file=ContentFile(data, content_hash),
+                    mime_type=mime_type or 'application/octet-stream',
+                    hash=content_hash,
+                )
+            result[name] = file_obj
+        except Exception as error:
+            print(f'Skipping file "{name}" during import: {error}')
+    return result
+
+
+
+def import_user_data(user, data):
+    """Fault-tolerant import of an export zip produced by `user_data()`.
+
+    Any of 'inventory.csv', 'friends.csv', 'locations.csv' or the 'files/' subfolder may be
+    missing; whatever is present is imported and everything else is silently skipped.
+    """
+    import io
+    import zipfile
+
+    summary = {'locations': 0, 'friends': 0, 'inventory_items': 0, 'files': 0}
+
+    with zipfile.ZipFile(io.BytesIO(data), 'r') as zip_file:
+        names = set(zip_file.namelist())
+
+        available_files = import_files(zip_file)
+        summary['files'] = len(available_files)
+
+        if 'locations.csv' in names:
+            summary['locations'] = import_locations(user, zip_file.read('locations.csv'))
+
+        if 'friends.csv' in names:
+            summary['friends'] = import_friends(user, zip_file.read('friends.csv'))
+
+        if 'inventory.csv' in names:
+            summary['inventory_items'] = import_inventory(user, zip_file.read('inventory.csv'), available_files)
+
+    return summary
+
+
+def _extract_zip_bytes(zip_payload):
+    """Normalize the incoming 'zip' request payload (uploaded file, base64 string, or raw bytes)."""
+    import base64
+
+    if hasattr(zip_payload, 'read'):
+        return zip_payload.read()
+    if isinstance(zip_payload, str):
+        try:
+            return base64.b64decode(zip_payload, validate=True)
+        except Exception:
+            return zip_payload.encode('utf-8')
+    return zip_payload
+
+
 @api_view(['POST'])
 @permission_classes([IsAuthenticated])
 @authentication_classes([SignatureAuthentication])
 def import_data(request, format=None):
-    zip = request.data.get('zip')
+    local_user = local_user_or_none(request.user)
-    if not zip:
+    if local_user is None:
+        return Response({'detail': 'This endpoint is only available to local users'}, status=403)
+    zip_payload = request.data.get('zip')
+    if not zip_payload:
         return Response(status=400)
-    for file_name, data in parse_user_data(zip):
+    try:
-        print(file_name, data)
+        zip_bytes = _extract_zip_bytes(zip_payload)
-    return Response(status=200)
+        summary = import_user_data(local_user, zip_bytes)
+    except Exception as error:
+        return Response({'detail': f'Could not read zip file: {error}'}, status=400)
+    return Response(summary, status=200)
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 @authentication_classes([SignatureAuthentication])
 def export_data(request, format=None):
-    return HttpResponse(user_data(), content_type='application/zip', status=200)
+    local_user = local_user_or_none(request.user)
+    if local_user is None:
+        return Response({'detail': 'This endpoint is only available to local users'}, status=403)
+    return HttpResponse(user_data(local_user), content_type='application/zip', status=200)
 
 
 @api_view(['DELETE'])
 @permission_classes([IsAuthenticated])
 @authentication_classes([SignatureAuthentication])
 def delete_account(request, format=None):
-    pass
+    local_user = local_user_or_none(request.user)
+    if local_user is None:
+        return Response({'detail': 'This endpoint is only available to local users'}, status=403)
 
 
 urlpatterns = [

backend/toolshed/offlinedata.py

@@ -0,0 +1,312 @@
+"""Data helpers for building/importing a user's offline export.
+
+These functions deal with the toolshed models and CSV row shapes only - they know nothing
+about the zip container format or how the request is authenticated. See toolshed/api/offlinedata.py
+for the zip-building/parsing and the API views themselves.
+"""
+
+
+def inventory_rows(user):
+    """Generator that yields the given user's inventory items as flattened dicts, one per row."""
+    import mimetypes
+
+    from toolshed.models import InventoryItem
+
+    items = (InventoryItem.objects
+             .filter(owner=user)
+             .select_related('category', 'storage_location')
+             .prefetch_related('tags', 'itemproperty_set__property', 'files'))
+
+    for item in items:
+        file_paths = []
+        for f in item.files.all():
+            extension = mimetypes.guess_extension(f.mime_type) or ''
+            file_paths.append(f'files/{f.hash}{extension}')
+
+        yield {
+            'id': item.id,
+            'name': item.name or '',
+            'description': item.description or '',
+            'category': str(item.category) if item.category else '',
+            'availability_policy': item.availability_policy,
+            'owned_quantity': item.owned_quantity,
+            'storage_location': str(item.storage_location) if item.storage_location else '',
+            'tags': ', '.join(tag.name for tag in item.tags.all()),
+            'properties': ', '.join(f"{ip.property.name}={ip.value}" for ip in item.itemproperty_set.all()),
+            'files': ', '.join(file_paths),
+            'created_at': item.created_at.isoformat() if item.created_at else '',
+        }
+
+
+def friend_rows(user):
+    """Generator that yields the given user's friends (known identities) as flattened dicts, one per row."""
+    friends = user.friends.all()
+
+    for friend in friends:
+        yield {
+            'username': friend.username,
+            'domain': friend.domain,
+            'handle': f'{friend.username}@{friend.domain}',
+            'public_key': friend.public_key,
+        }
+
+
+def location_path(location):
+    """Recursively build the '/'-joined path of a StorageLocation, following its parent chain."""
+    if location.parent:
+        return location_path(location.parent) + '/' + location.name
+    return location.name
+
+
+def location_rows(user):
+    """Generator that yields the given user's storage locations as flattened dicts, one per row."""
+    from toolshed.models import StorageLocation
+
+    locations = StorageLocation.objects.filter(owner=user).select_related('category', 'parent')
+
+    for location in locations:
+        yield {
+            'id': location.id,
+            'name': location.name,
+            'description': location.description or '',
+            'category': str(location.category) if location.category else '',
+            'parent': str(location.parent) if location.parent else '',
+            'path': location_path(location),
+        }
+
+
+def inventory_files(user):
+    """Generator that yields (arcname, data) for each unique file attached to the user's inventory items.
+
+    Files are deduplicated by hash and placed under a 'files/' subfolder, keeping their original
+    extension (guessed from mime_type) so attachments and images remain viewable once extracted.
+    """
+    import mimetypes
+
+    from toolshed.models import InventoryItem
+
+    seen_hashes = set()
+    items = InventoryItem.objects.filter(owner=user).prefetch_related('files')
+
+    for item in items:
+        for file in item.files.all():
+            if file.hash in seen_hashes:
+                continue
+            seen_hashes.add(file.hash)
+
+            extension = mimetypes.guess_extension(file.mime_type) or ''
+            arcname = f'files/{file.hash}{extension}'
+
+            file.file.open('rb')
+            try:
+                data = file.file.read()
+            finally:
+                file.file.close()
+
+            yield arcname, data
+
+
+def rows_to_csv(rows, fieldnames=None, encoding='utf-8'):
+    """Generator that consumes an iterable of dicts and yields encoded CSV data chunk by chunk."""
+    import csv
+
+    class _Echo:
+        """A file-like object whose write() just returns what was passed, for streaming csv.writer output."""
+
+        def write(self, value):
+            return value
+
+    rows = iter(rows)
+    try:
+        first_row = next(rows)
+    except StopIteration:
+        return
+
+    if fieldnames is None:
+        fieldnames = list(first_row.keys())
+
+    writer = csv.DictWriter(_Echo(), fieldnames=fieldnames)
+    yield writer.writeheader().encode(encoding)
+    yield writer.writerow(first_row).encode(encoding)
+    for row in rows:
+        yield writer.writerow(row).encode(encoding)
+
+
+def _read_csv_rows(data, encoding='utf-8'):
+    """Decode CSV bytes and yield rows as dicts keyed by the header labels (not column offsets)."""
+    import csv
+    import io
+
+    reader = csv.DictReader(io.StringIO(data.decode(encoding)))
+    for row in reader:
+        yield row
+
+
+def _field_is_optional(model, field_name):
+    """Return True if a field may be omitted: it's a relation, has a default, or allows null/blank."""
+    try:
+        field = model._meta.get_field(field_name)
+    except Exception:
+        return True
+    if getattr(field, 'many_to_many', False) or getattr(field, 'one_to_many', False):
+        return True
+    return bool(getattr(field, 'null', False) or getattr(field, 'blank', False) or field.has_default())
+
+
+def get_or_create_category(path):
+    """Resolve or create a Category from a '/'-separated path such as 'Electronics/Cables'."""
+    from toolshed.models import Category
+
+    parent = None
+    category = None
+    for part in (p for p in path.split('/') if p):
+        category, _ = Category.objects.get_or_create(name=part, parent=parent, defaults={'origin': 'import'})
+        parent = category
+    return category
+
+
+def import_locations(user, data):
+    """Fault-tolerant import of locations.csv into StorageLocations owned by `user`.
+
+    Rows are read by header label. Rows missing the required 'name' column are skipped.
+    Optional columns ('description', 'category') are simply omitted if absent. Locations are
+    processed in path-depth order so a child's parent already exists by the time it's needed.
+    """
+    from toolshed.models import StorageLocation
+
+    rows = list(_read_csv_rows(data))
+    rows.sort(key=lambda row: (row.get('path') or row.get('name') or '').count('/'))
+
+    resolved_by_path = {}
+    imported = 0
+
+    for row in rows:
+        try:
+            name = (row.get('name') or '').strip()
+            if not name and not _field_is_optional(StorageLocation, 'name'):
+                continue  # required column missing, skip this row
+
+            path = (row.get('path') or name).strip()
+            parent = None
+            if '/' in path:
+                parent = resolved_by_path.get(path.rsplit('/', 1)[0])
+
+            category = None
+            category_path = row.get('category')
+            if category_path:
+                category = get_or_create_category(category_path)
+
+            location, _ = StorageLocation.objects.update_or_create(
+                owner=user, name=name, parent=parent,
+                defaults={
+                    'description': row.get('description', '') or '',
+                    'category': category,
+                },
+            )
+            resolved_by_path[path] = location
+            imported += 1
+        except Exception as error:
+            print(f'Skipping location row {row}: {error}')
+
+    return imported
+
+
+def import_friends(user, data):
+    """Fault-tolerant import of friends.csv, adding valid entries to the user's known friends."""
+    from authentication.models import KnownIdentity
+
+    imported = 0
+    for row in _read_csv_rows(data):
+        try:
+            username = row.get('username')
+            domain = row.get('domain')
+            handle = row.get('handle')
+            if (not username or not domain) and handle and '@' in handle:
+                username, domain = handle.split('@', 1)
+
+            public_key = row.get('public_key')
+            if not username or not domain or not public_key:
+                continue  # required fields missing, skip this row
+
+            identity, _ = KnownIdentity.objects.get_or_create(
+                username=username, domain=domain,
+                defaults={'public_key': public_key},
+            )
+            user.public_identity.friends.add(identity)
+            imported += 1
+        except Exception as error:
+            print(f'Skipping friend row {row}: {error}')
+
+    return imported
+
+
+def import_inventory(user, data, available_files):
+    """Fault-tolerant import of inventory.csv into InventoryItems owned by `user`.
+
+    `available_files` maps the 'files' column's paths to File instances successfully extracted
+    from the zip; references to missing files are ignored rather than failing the row.
+    """
+    from toolshed.models import InventoryItem, ItemProperty, StorageLocation, Tag, Property
+
+    imported = 0
+    for row in _read_csv_rows(data):
+        try:
+            name = (row.get('name') or '').strip()
+
+            file_paths = [p.strip() for p in (row.get('files') or '').split(',') if p.strip()]
+            files = [available_files[p] for p in file_paths if p in available_files]
+
+            if not name and not files:
+                continue  # InventoryItem requires a name or at least one file
+
+            category = None
+            category_path = row.get('category')
+            if category_path:
+                category = get_or_create_category(category_path)
+
+            storage_location = None
+            location_path_value = row.get('storage_location')
+            if location_path_value:
+                storage_location = StorageLocation.objects.filter(
+                    owner=user, name=location_path_value.rsplit('/', 1)[-1]).first()
+
+            try:
+                owned_quantity = int(row.get('owned_quantity') or 1)
+            except (TypeError, ValueError):
+                owned_quantity = 1
+
+            item = InventoryItem.objects.create(
+                owner=user,
+                name=name or None,
+                description=row.get('description', '') or '',
+                category=category,
+                availability_policy=row.get('availability_policy') or 'private',
+                owned_quantity=owned_quantity,
+                storage_location=storage_location,
+            )
+
+            for tag_name in (row.get('tags') or '').split(','):
+                tag_name = tag_name.strip()
+                if tag_name:
+                    tag, _ = Tag.objects.get_or_create(name=tag_name, category=None,
+                                                       defaults={'origin': 'import'})
+                    item.tags.add(tag, through_defaults={})
+
+            for prop_entry in (row.get('properties') or '').split(','):
+                prop_entry = prop_entry.strip()
+                if '=' not in prop_entry:
+                    continue
+                prop_name, value = prop_entry.split('=', 1)
+                prop, _ = Property.objects.get_or_create(name=prop_name.strip(), category=None,
+                                                          defaults={'origin': 'import'})
+                ItemProperty.objects.create(inventory_item=item, property=prop, value=value.strip())
+
+            for file in files:
+                item.files.add(file)
+
+            imported += 1
+        except Exception as error:
+            print(f'Skipping inventory row {row}: {error}')
+
+    return imported
+

frontend/src/views/settings/Data.vue

@@ -103,20 +103,50 @@ export default {
         exportKey() {
             const key = this.userIdentityRecord;
         },
+        fileToBase64(file) {
+            return new Promise((resolve, reject) => {
+                const reader = new FileReader();
+                reader.onload = () => {
+                    const buffer = reader.result;
+                    if (!(buffer instanceof ArrayBuffer)) {
+                        reject(new Error('Could not read file'));
+                        return;
+                    }
+                    const data = new Uint8Array(buffer);
+                    const base64 = btoa(data.reduce((acc, byte) => acc + String.fromCharCode(byte), ''));
+                    resolve(base64);
+                };
+                reader.onerror = (error) => reject(error);
+                reader.readAsArrayBuffer(file);
+            });
+        },
         async importData() {
             if (!this.selectedFile) {
                 alert('Please select a file to import');
                 return;
             }
-            alert('Data import not implemented');
+            try {
-            return;
+                const base64 = await this.fileToBase64(this.selectedFile);
-
+                const servers = await this.getHomeServers();
-            const formData = new FormData();
+                const summary = await servers.post(this.signAuth, '/api/import/', {zip: base64});
-            formData.append('file', this.selectedFile);
+                if (summary && summary.detail) {
-            console.log(formData);
+                    alert('Data import failed: ' + summary.detail);
-            const servers = await this.getHomeServers();
+                    return;
-            const data = await servers.postRaw(this.signHashedAuth, '/api/import/', formData);
+                }
-            console.log(data);
+                alert('Data imported successfully: ' +
+                    `${summary.inventory_items || 0} inventory items, ` +
+                    `${summary.friends || 0} friends, ` +
+                    `${summary.locations || 0} locations, ` +
+                    `${summary.files || 0} files.`);
+                this.selectedFile = null;
+                const fileInput = document.getElementById('inputFile');
+                if (fileInput) {
+                    fileInput.value = '';
+                }
+            } catch (error) {
+                console.error('Data import failed', error);
+                alert('Data import failed: ' + error);
+            }
         },
         async exportData() {
             const servers = await this.getHomeServers();

testdata/user-a.key

@@ -0,0 +1 @@
+f0cd0b223efb7ca24cd8ad6e39471fb1b4f69a20a3e5c6fdb41c9985b7dbf381

testdata/user-b.key

@@ -0,0 +1 @@
+6b186fde49d12b7108e5246b5279be660fba185cfd514cb0d949fea5d7db9131