578 lines
22 KiB
Python
578 lines
22 KiB
Python
"""Data helpers for building/importing a user's offline export; deal with toolshed models and CSV row shapes only, and know nothing about the zip container format or request auth - see toolshed/api/offlinedata.py for that."""
|
|
|
|
|
|
def inventory_rows(user):
|
|
"""Generator that yields the given user's inventory items as flattened dicts, one per row."""
|
|
import mimetypes
|
|
|
|
from toolshed.models import InventoryItem
|
|
|
|
items = (InventoryItem.objects
|
|
.filter(owner=user)
|
|
.select_related('category', 'storage_location')
|
|
.prefetch_related('tags', 'itemproperty_set__property', 'files'))
|
|
|
|
for item in items:
|
|
file_paths = []
|
|
for f in item.files.all():
|
|
extension = mimetypes.guess_extension(f.mime_type) or ''
|
|
file_paths.append(f'files/{f.hash}{extension}')
|
|
|
|
yield {
|
|
'id': item.id,
|
|
'name': item.name or '',
|
|
'description': item.description or '',
|
|
'category': item.category.get_handle() if item.category else '',
|
|
'availability_policy': item.availability_policy,
|
|
'owned_quantity': item.owned_quantity,
|
|
'storage_location': str(item.storage_location) if item.storage_location else '',
|
|
'tags': ', '.join(tag.get_handle() for tag in item.tags.all()),
|
|
'properties': _encode_properties_cell(item.itemproperty_set.all()),
|
|
'files': ', '.join(file_paths),
|
|
'created_at': item.created_at.isoformat() if item.created_at else '',
|
|
}
|
|
|
|
|
|
def friend_rows(user):
|
|
"""Generator that yields the given user's friends (known identities) as flattened dicts, one per row."""
|
|
friends = user.friends.all()
|
|
|
|
for friend in friends:
|
|
yield {
|
|
'username': friend.username,
|
|
'domain': friend.domain,
|
|
'handle': f'{friend.username}@{friend.domain}',
|
|
'public_key': friend.public_key,
|
|
}
|
|
|
|
|
|
def location_path(location):
|
|
"""Recursively build the '/'-joined path of a StorageLocation, following its parent chain."""
|
|
if location.parent:
|
|
return location_path(location.parent) + '/' + location.name
|
|
return location.name
|
|
|
|
|
|
def location_rows(user):
|
|
"""Generator that yields the given user's storage locations as flattened dicts, one per row."""
|
|
from toolshed.models import StorageLocation
|
|
|
|
locations = StorageLocation.objects.filter(owner=user).select_related('category', 'parent')
|
|
|
|
for location in locations:
|
|
yield {
|
|
'id': location.id,
|
|
'name': location.name,
|
|
'description': location.description or '',
|
|
'category': str(location.category) if location.category else '',
|
|
'parent': str(location.parent) if location.parent else '',
|
|
'path': location_path(location),
|
|
}
|
|
|
|
|
|
def inventory_files(user):
|
|
"""Yields (arcname, data) for each unique file attached to the user's inventory items, deduplicated by hash. See docs/implementation.md#file-naming-convention-in-exports for the naming scheme."""
|
|
import mimetypes
|
|
|
|
from toolshed.models import InventoryItem
|
|
|
|
seen_hashes = set()
|
|
items = InventoryItem.objects.filter(owner=user).prefetch_related('files')
|
|
|
|
for item in items:
|
|
for file in item.files.all():
|
|
if file.hash in seen_hashes:
|
|
continue
|
|
seen_hashes.add(file.hash)
|
|
|
|
extension = mimetypes.guess_extension(file.mime_type) or ''
|
|
arcname = f'files/{file.hash}{extension}'
|
|
|
|
file.file.open('rb')
|
|
try:
|
|
data = file.file.read()
|
|
finally:
|
|
file.file.close()
|
|
|
|
yield arcname, data
|
|
|
|
|
|
def profile_data(user):
|
|
"""Return the given user's profile as a dict, matching profile.json in the export zip."""
|
|
import mimetypes
|
|
|
|
data = {
|
|
'username': user.username,
|
|
'domain': user.domain,
|
|
'email': user.email,
|
|
'first_name': user.first_name or '',
|
|
'last_name': user.last_name or '',
|
|
'profile_picture': None,
|
|
}
|
|
if user.profile_picture:
|
|
extension = mimetypes.guess_extension(user.profile_picture.mime_type) or ''
|
|
data['profile_picture'] = f'files/{user.profile_picture.hash}{extension}'
|
|
return data
|
|
|
|
|
|
def profile_picture_files(user):
|
|
"""Generator that yields (arcname, data) for the user's profile picture, if one is set. Kept separate from `inventory_files()` so the picture is included even for users with no inventory items or whose picture isn't attached to any item."""
|
|
import mimetypes
|
|
|
|
if not user.profile_picture:
|
|
return
|
|
|
|
extension = mimetypes.guess_extension(user.profile_picture.mime_type) or ''
|
|
arcname = f'files/{user.profile_picture.hash}{extension}'
|
|
|
|
user.profile_picture.file.open('rb')
|
|
try:
|
|
data = user.profile_picture.file.read()
|
|
finally:
|
|
user.profile_picture.file.close()
|
|
|
|
yield arcname, data
|
|
|
|
|
|
def settings_data(user):
|
|
"""Return the given user's account-level preferences as a {key: value} dict for settings.json. Only account preferences are exported; device-level preferences stay in the browser's local storage since they describe the device, not the account."""
|
|
return {pref.key: pref.value for pref in user.preferences.all()}
|
|
|
|
|
|
def import_profile(user, data, available_files):
|
|
"""Fault-tolerant import of profile.json, updating the user's editable profile fields. See docs/implementation.md#profile-import-semantics for which fields are applied and why."""
|
|
import json
|
|
|
|
try:
|
|
profile = json.loads(data.decode('utf-8'))
|
|
if not isinstance(profile, dict):
|
|
return False
|
|
|
|
if 'first_name' in profile:
|
|
user.first_name = profile.get('first_name') or ''
|
|
if 'last_name' in profile:
|
|
user.last_name = profile.get('last_name') or ''
|
|
if profile.get('email'):
|
|
user.email = profile['email']
|
|
|
|
picture_path = profile.get('profile_picture')
|
|
if picture_path and picture_path in available_files:
|
|
user.profile_picture = available_files[picture_path]
|
|
|
|
user.save()
|
|
return True
|
|
except Exception as error:
|
|
print(f'Skipping profile.json: {error}')
|
|
return False
|
|
|
|
|
|
def import_settings(user, data):
|
|
"""Fault-tolerant import of settings.json; each top-level key/value pair is upserted as an AccountPreference, and unreadable data or a non-object payload is skipped rather than aborting the whole import."""
|
|
import json
|
|
|
|
from authentication.models import AccountPreference
|
|
|
|
try:
|
|
settings = json.loads(data.decode('utf-8'))
|
|
if not isinstance(settings, dict):
|
|
return 0
|
|
|
|
imported = 0
|
|
for key, value in settings.items():
|
|
try:
|
|
AccountPreference.objects.update_or_create(user=user, key=key, defaults={'value': value})
|
|
imported += 1
|
|
except Exception as error:
|
|
print(f'Skipping setting "{key}": {error}')
|
|
return imported
|
|
except Exception as error:
|
|
print(f'Skipping settings.json: {error}')
|
|
return 0
|
|
|
|
|
|
def delete_user_data(user):
|
|
"""Permanently delete everything that `user_data()` exports, keeping the account itself intact. Returns a summary dict describing what was removed. See docs/implementation.md#account-data-deletion for exactly what's removed and why the account itself survives."""
|
|
from django.db import transaction
|
|
|
|
from toolshed.models import InventoryItem, StorageLocation
|
|
|
|
summary = {'inventory_items': 0, 'locations': 0, 'settings': 0, 'friends': 0, 'files': 0}
|
|
|
|
with transaction.atomic():
|
|
candidate_file_ids = set(
|
|
InventoryItem.global_objects.filter(owner=user).values_list('files__id', flat=True))
|
|
candidate_file_ids.discard(None)
|
|
if user.profile_picture_id:
|
|
candidate_file_ids.add(user.profile_picture_id)
|
|
|
|
for item in InventoryItem.global_objects.filter(owner=user):
|
|
item.hard_delete()
|
|
summary['inventory_items'] += 1
|
|
|
|
summary['locations'], _ = StorageLocation.objects.filter(owner=user).delete()
|
|
summary['settings'], _ = user.preferences.all().delete()
|
|
|
|
summary['friends'] = user.public_identity.friends.count()
|
|
user.public_identity.friends.clear()
|
|
|
|
user.profile_picture = None
|
|
user.save(update_fields=['profile_picture'])
|
|
|
|
summary['files'] = _delete_orphaned_files(candidate_file_ids)
|
|
|
|
|
|
return summary
|
|
|
|
|
|
def delete_user_account(user):
|
|
"""Permanently delete the local user's account, after wiping all of its data via `delete_user_data()`. Returns a summary dict with `account` set to True. See docs/implementation.md#account-data-deletion for why the underlying KnownIdentity is kept."""
|
|
from django.db import transaction
|
|
|
|
with transaction.atomic():
|
|
summary = delete_user_data(user)
|
|
user.delete()
|
|
summary['account'] = True
|
|
|
|
return summary
|
|
|
|
|
|
def _delete_orphaned_files(file_ids):
|
|
"""Delete File rows (and their underlying blobs) in `file_ids` that are no longer referenced. Returns the number of files deleted. See docs/implementation.md#account-data-deletion for the orphan definition."""
|
|
from files.models import File
|
|
|
|
deleted = 0
|
|
for file_obj in File.objects.filter(id__in=file_ids):
|
|
if file_obj.connected_items.exists() or file_obj.profile_picture_users.exists() \
|
|
or file_obj.staged_by_workflows.exists():
|
|
continue
|
|
file_obj.file.delete(save=False)
|
|
file_obj.delete()
|
|
deleted += 1
|
|
return deleted
|
|
|
|
|
|
def rows_to_csv(rows, fieldnames=None, encoding='utf-8'):
|
|
"""Generator that consumes an iterable of dicts and yields encoded CSV data chunk by chunk."""
|
|
import csv
|
|
|
|
class _Echo:
|
|
"""A file-like object whose write() just returns what was passed, for streaming csv.writer output."""
|
|
|
|
def write(self, value):
|
|
return value
|
|
|
|
rows = iter(rows)
|
|
try:
|
|
first_row = next(rows)
|
|
except StopIteration:
|
|
return
|
|
|
|
if fieldnames is None:
|
|
fieldnames = list(first_row.keys())
|
|
|
|
writer = csv.DictWriter(_Echo(), fieldnames=fieldnames)
|
|
yield writer.writeheader().encode(encoding)
|
|
yield writer.writerow(first_row).encode(encoding)
|
|
for row in rows:
|
|
yield writer.writerow(row).encode(encoding)
|
|
|
|
|
|
def _read_csv_rows(data, encoding='utf-8'):
|
|
"""Decode CSV bytes and yield rows as dicts keyed by the header labels (not column offsets)."""
|
|
import csv
|
|
import io
|
|
|
|
reader = csv.DictReader(io.StringIO(data.decode(encoding)))
|
|
for row in reader:
|
|
yield row
|
|
|
|
|
|
def _field_is_optional(model, field_name):
|
|
"""Return True if a field may be omitted: it's a relation, has a default, or allows null/blank."""
|
|
try:
|
|
field = model._meta.get_field(field_name)
|
|
except Exception:
|
|
return True
|
|
if getattr(field, 'many_to_many', False) or getattr(field, 'one_to_many', False):
|
|
return True
|
|
return bool(getattr(field, 'null', False) or getattr(field, 'blank', False) or field.has_default())
|
|
|
|
|
|
def get_or_create_category(path):
|
|
"""Resolve or create a Category from a '/'-separated path such as 'Electronics/Cables'."""
|
|
from toolshed.models import Category
|
|
|
|
parent = None
|
|
category = None
|
|
for part in (p for p in path.split('/') if p):
|
|
category, _ = Category.objects.get_or_create(name=part, parent=parent, defaults={'origin': 'import'})
|
|
parent = category
|
|
return category
|
|
|
|
|
|
def import_locations(user, data):
|
|
"""Fault-tolerant import of locations.csv into StorageLocations owned by `user`. See docs/implementation.md#location-import-ordering-and-savepoints for row ordering and error-isolation rules."""
|
|
from django.db import transaction
|
|
|
|
from toolshed.models import StorageLocation
|
|
|
|
rows = list(_read_csv_rows(data))
|
|
rows.sort(key=lambda row: (row.get('path') or row.get('name') or '').count('/'))
|
|
|
|
resolved_by_path = {}
|
|
imported = 0
|
|
|
|
for row in rows:
|
|
try:
|
|
with transaction.atomic():
|
|
name = (row.get('name') or '').strip()
|
|
if not name and not _field_is_optional(StorageLocation, 'name'):
|
|
continue # required column missing, skip this row
|
|
|
|
path = (row.get('path') or name).strip()
|
|
parent = None
|
|
if '/' in path:
|
|
parent = resolved_by_path.get(path.rsplit('/', 1)[0])
|
|
|
|
category = None
|
|
category_path = row.get('category')
|
|
if category_path:
|
|
category = get_or_create_category(category_path)
|
|
|
|
defaults = {
|
|
'description': row.get('description', '') or '',
|
|
'category': category,
|
|
}
|
|
try:
|
|
location = StorageLocation.objects.get(owner=user, name=name, parent=parent)
|
|
for field, value in defaults.items():
|
|
setattr(location, field, value)
|
|
location.save(update_fields=list(defaults.keys()))
|
|
except StorageLocation.DoesNotExist:
|
|
location = StorageLocation.create_for_owner(owner=user, name=name, parent=parent, **defaults)
|
|
resolved_by_path[path] = location
|
|
imported += 1
|
|
except Exception as error:
|
|
print(f'Skipping location row {row}: {error}')
|
|
|
|
return imported
|
|
|
|
|
|
def import_friends(user, data):
|
|
"""Fault-tolerant import of friends.csv, adding valid entries to the user's known friends."""
|
|
from django.db import transaction
|
|
|
|
from authentication.models import KnownIdentity
|
|
|
|
imported = 0
|
|
for row in _read_csv_rows(data):
|
|
try:
|
|
with transaction.atomic():
|
|
username = row.get('username')
|
|
domain = row.get('domain')
|
|
handle = row.get('handle')
|
|
if (not username or not domain) and handle and '@' in handle:
|
|
username, domain = handle.split('@', 1)
|
|
|
|
public_key = row.get('public_key')
|
|
if not username or not domain or not public_key:
|
|
continue # required fields missing, skip this row
|
|
|
|
identity, _ = KnownIdentity.objects.get_or_create(
|
|
username=username, domain=domain,
|
|
defaults={'public_key': public_key},
|
|
)
|
|
user.public_identity.friends.add(identity)
|
|
imported += 1
|
|
except Exception as error:
|
|
print(f'Skipping friend row {row}: {error}')
|
|
|
|
return imported
|
|
|
|
|
|
class _HandleNotFound(Exception):
|
|
"""Raised when a fully qualified handle (e.g. 'git:base#tag:drill') can't be resolved, aborting the row rather than creating a new entity. See docs/implementation.md#handle-resolution-semantics."""
|
|
|
|
|
|
def _resolve_handle(value, entity_type, model):
|
|
"""Resolve a fully qualified handle (e.g. 'git:base#tag:drill') to an *existing* model instance; never creates one. See docs/implementation.md#handle-resolution-semantics for the rationale."""
|
|
origin, rest = value.split('#', 1)
|
|
if ':' in rest:
|
|
found_type, name = rest.split(':', 1)
|
|
if found_type != entity_type:
|
|
raise _HandleNotFound(
|
|
f"expected a {entity_type} handle but got '{value}' (type '{found_type}')")
|
|
else:
|
|
name = rest
|
|
try:
|
|
return model.objects.get(origin=origin, name=name)
|
|
except model.DoesNotExist:
|
|
raise _HandleNotFound(f"{entity_type} '{value}' does not exist, skipping item")
|
|
|
|
|
|
def _quote_value_if_needed(value):
|
|
"""Wrap `value` in double quotes (CSV-style, doubling embedded quotes) if it contains a comma or quote character. See docs/implementation.md#properties-csv-encoding."""
|
|
if any(ch in value for ch in ',"'):
|
|
return '"' + value.replace('"', '""') + '"'
|
|
return value
|
|
|
|
|
|
def _encode_properties_cell(item_properties):
|
|
"""Encode an item's properties as a comma-separated "handle=value" list for the 'properties' CSV cell. See docs/implementation.md#properties-csv-encoding and `_parse_properties_cell()` for the reader side."""
|
|
entries = [
|
|
f"{ip.property.get_handle()}={_quote_value_if_needed(ip.value or '')}"
|
|
for ip in item_properties
|
|
]
|
|
return ', '.join(entries)
|
|
|
|
|
|
def _split_quoted_comma_list(raw_value):
|
|
"""Split a comma-separated list into entries, honouring double-quoted substrings (CSV-style) so a quoted value's own commas aren't mistaken for separators. See docs/implementation.md#properties-csv-encoding for the quoting/whitespace rules this implements."""
|
|
entries = []
|
|
current = []
|
|
in_quotes = False
|
|
i = 0
|
|
length = len(raw_value)
|
|
|
|
while i < length:
|
|
char = raw_value[i]
|
|
if char == '"':
|
|
if in_quotes and i + 1 < length and raw_value[i + 1] == '"':
|
|
current.append('"')
|
|
i += 2
|
|
continue
|
|
in_quotes = not in_quotes
|
|
i += 1
|
|
continue
|
|
if char == ',' and not in_quotes:
|
|
entries.append(''.join(current))
|
|
current = []
|
|
i += 1
|
|
if i < length and raw_value[i] == ' ':
|
|
i += 1 # skip the single space of the ", " separator
|
|
continue
|
|
current.append(char)
|
|
i += 1
|
|
|
|
entries.append(''.join(current))
|
|
return [entry for entry in entries if entry]
|
|
|
|
|
|
def _parse_properties_cell(raw_value, resolve_property):
|
|
"""Parse the 'properties' CSV cell into a list of (Property, value) tuples. See docs/implementation.md#properties-csv-encoding for how the cell is encoded by `_encode_properties_cell()`."""
|
|
raw_value = (raw_value or '').strip()
|
|
if not raw_value:
|
|
return []
|
|
|
|
properties = []
|
|
for prop_entry in _split_quoted_comma_list(raw_value):
|
|
if '=' not in prop_entry:
|
|
continue
|
|
prop_name, value = prop_entry.split('=', 1)
|
|
prop = resolve_property(prop_name.strip())
|
|
if prop:
|
|
properties.append((prop, value))
|
|
|
|
return properties
|
|
|
|
|
|
|
|
def import_inventory(user, data, available_files):
|
|
"""Fault-tolerant import of inventory.csv into InventoryItems owned by `user`. See docs/implementation.md#inventory-import-semantics for file/handle resolution and error-isolation rules."""
|
|
from django.db import transaction
|
|
|
|
from toolshed.models import Category, InventoryItem, ItemProperty, StorageLocation, Tag, Property
|
|
|
|
def resolve_category_from_csv(value):
|
|
"""Resolve a category from CSV - supports both old format and fully qualified handles"""
|
|
if not value:
|
|
return None
|
|
if '#' in value:
|
|
return _resolve_handle(value, 'category', Category)
|
|
# Fallback to old path-based lookup
|
|
return get_or_create_category(value)
|
|
|
|
def resolve_tag_from_csv(value):
|
|
"""Resolve a tag from CSV - supports both old format and fully qualified handles"""
|
|
if not value:
|
|
return None
|
|
if '#' in value:
|
|
return _resolve_handle(value, 'tag', Tag)
|
|
# Fallback to old name-only lookup or create
|
|
return Tag.objects.get_or_create(name=value, category=None, defaults={'origin': 'import'})[0]
|
|
|
|
def resolve_property_from_csv(value):
|
|
"""Resolve a property from CSV - supports both old format and fully qualified handles"""
|
|
if not value:
|
|
return None
|
|
if '#' in value:
|
|
return _resolve_handle(value, 'property', Property)
|
|
# Fallback to old name-only lookup or create
|
|
return Property.objects.get_or_create(name=value, category=None, defaults={'origin': 'import'})[0]
|
|
|
|
imported = 0
|
|
errors = []
|
|
for row in _read_csv_rows(data):
|
|
name = (row.get('name') or '').strip()
|
|
try:
|
|
with transaction.atomic():
|
|
file_paths = [p.strip() for p in (row.get('files') or '').split(',') if p.strip()]
|
|
files = [available_files[p] for p in file_paths if p in available_files]
|
|
|
|
if not name and not files:
|
|
continue # InventoryItem requires a name or at least one file
|
|
|
|
category = resolve_category_from_csv((row.get('category') or '').strip())
|
|
|
|
tags = []
|
|
for tag_name in (row.get('tags') or '').split(','):
|
|
tag_name = tag_name.strip()
|
|
if tag_name:
|
|
tag = resolve_tag_from_csv(tag_name)
|
|
if tag:
|
|
tags.append(tag)
|
|
|
|
properties = _parse_properties_cell(row.get('properties') or '', resolve_property_from_csv)
|
|
|
|
storage_location = None
|
|
location_path_value = row.get('storage_location')
|
|
if location_path_value:
|
|
storage_location = StorageLocation.objects.filter(
|
|
owner=user, name=location_path_value.rsplit('/', 1)[-1]).first()
|
|
|
|
try:
|
|
owned_quantity = int(row.get('owned_quantity') or 1)
|
|
except (TypeError, ValueError):
|
|
owned_quantity = 1
|
|
|
|
item = InventoryItem.create_for_owner(
|
|
owner=user,
|
|
name=name or None,
|
|
description=row.get('description', '') or '',
|
|
category=category,
|
|
availability_policy=row.get('availability_policy') or 'private',
|
|
owned_quantity=owned_quantity,
|
|
storage_location=storage_location,
|
|
)
|
|
|
|
for tag in tags:
|
|
item.tags.add(tag, through_defaults={})
|
|
|
|
for prop, value in properties:
|
|
ItemProperty.objects.create(inventory_item=item, property=prop, value=value)
|
|
|
|
for file in files:
|
|
item.files.add(file)
|
|
|
|
imported += 1
|
|
except _HandleNotFound as error:
|
|
message = f"Skipping item '{name or 'unnamed'}': {error}"
|
|
print(message)
|
|
errors.append(message)
|
|
except Exception as error:
|
|
message = f'Skipping inventory row {row}: {error}'
|
|
print(message)
|
|
errors.append(message)
|
|
|
|
return imported, errors
|
|
|