toolshed/backend/toolshed/offlinedata.py
2026-08-23 14:07:52 +02:00

671 lines
25 KiB
Python

"""Data helpers for building/importing a user's offline export.
These functions deal with the toolshed models and CSV row shapes only - they know nothing
about the zip container format or how the request is authenticated. See toolshed/api/offlinedata.py
for the zip-building/parsing and the API views themselves.
"""
def inventory_rows(user):
"""Generator that yields the given user's inventory items as flattened dicts, one per row."""
import mimetypes
from toolshed.models import InventoryItem
items = (InventoryItem.objects
.filter(owner=user)
.select_related('category', 'storage_location')
.prefetch_related('tags', 'itemproperty_set__property', 'files'))
for item in items:
file_paths = []
for f in item.files.all():
extension = mimetypes.guess_extension(f.mime_type) or ''
file_paths.append(f'files/{f.hash}{extension}')
yield {
'id': item.id,
'name': item.name or '',
'description': item.description or '',
'category': item.category.get_handle() if item.category else '',
'availability_policy': item.availability_policy,
'owned_quantity': item.owned_quantity,
'storage_location': str(item.storage_location) if item.storage_location else '',
'tags': ', '.join(tag.get_handle() for tag in item.tags.all()),
'properties': _encode_properties_cell(item.itemproperty_set.all()),
'files': ', '.join(file_paths),
'created_at': item.created_at.isoformat() if item.created_at else '',
}
def friend_rows(user):
"""Generator that yields the given user's friends (known identities) as flattened dicts, one per row."""
friends = user.friends.all()
for friend in friends:
yield {
'username': friend.username,
'domain': friend.domain,
'handle': f'{friend.username}@{friend.domain}',
'public_key': friend.public_key,
}
def location_path(location):
"""Recursively build the '/'-joined path of a StorageLocation, following its parent chain."""
if location.parent:
return location_path(location.parent) + '/' + location.name
return location.name
def location_rows(user):
"""Generator that yields the given user's storage locations as flattened dicts, one per row."""
from toolshed.models import StorageLocation
locations = StorageLocation.objects.filter(owner=user).select_related('category', 'parent')
for location in locations:
yield {
'id': location.id,
'name': location.name,
'description': location.description or '',
'category': str(location.category) if location.category else '',
'parent': str(location.parent) if location.parent else '',
'path': location_path(location),
}
def inventory_files(user):
"""Generator that yields (arcname, data) for each unique file attached to the user's inventory items.
Files are deduplicated by hash and placed under a 'files/' subfolder, keeping their original
extension (guessed from mime_type) so attachments and images remain viewable once extracted.
"""
import mimetypes
from toolshed.models import InventoryItem
seen_hashes = set()
items = InventoryItem.objects.filter(owner=user).prefetch_related('files')
for item in items:
for file in item.files.all():
if file.hash in seen_hashes:
continue
seen_hashes.add(file.hash)
extension = mimetypes.guess_extension(file.mime_type) or ''
arcname = f'files/{file.hash}{extension}'
file.file.open('rb')
try:
data = file.file.read()
finally:
file.file.close()
yield arcname, data
def profile_data(user):
"""Return the given user's profile as a dict, matching profile.json in the export zip."""
import mimetypes
data = {
'username': user.username,
'domain': user.domain,
'email': user.email,
'first_name': user.first_name or '',
'last_name': user.last_name or '',
'profile_picture': None,
}
if user.profile_picture:
extension = mimetypes.guess_extension(user.profile_picture.mime_type) or ''
data['profile_picture'] = f'files/{user.profile_picture.hash}{extension}'
return data
def profile_picture_files(user):
"""Generator that yields (arcname, data) for the user's profile picture, if one is set.
Kept separate from `inventory_files()` so the profile picture is included in the export
even for users with no inventory items or whose picture isn't attached to any item.
"""
import mimetypes
if not user.profile_picture:
return
extension = mimetypes.guess_extension(user.profile_picture.mime_type) or ''
arcname = f'files/{user.profile_picture.hash}{extension}'
user.profile_picture.file.open('rb')
try:
data = user.profile_picture.file.read()
finally:
user.profile_picture.file.close()
yield arcname, data
def settings_data(user):
"""Return the given user's account-level preferences as a {key: value} dict for settings.json.
Only account preferences (AccountPreference) are exported; device-level preferences stay
in the browser's local storage since they describe the device, not the account.
"""
return {pref.key: pref.value for pref in user.preferences.all()}
def import_profile(user, data, available_files):
"""Fault-tolerant import of profile.json, updating the user's editable profile fields.
Only 'first_name', 'last_name', 'email' and 'profile_picture' (a 'files/...' path present in
`available_files`) are applied; 'username' and 'domain' are ignored since they identify the
account itself and can't be changed by an import.
"""
import json
try:
profile = json.loads(data.decode('utf-8'))
if not isinstance(profile, dict):
return False
if 'first_name' in profile:
user.first_name = profile.get('first_name') or ''
if 'last_name' in profile:
user.last_name = profile.get('last_name') or ''
if profile.get('email'):
user.email = profile['email']
picture_path = profile.get('profile_picture')
if picture_path and picture_path in available_files:
user.profile_picture = available_files[picture_path]
user.save()
return True
except Exception as error:
print(f'Skipping profile.json: {error}')
return False
def import_settings(user, data):
"""Fault-tolerant import of settings.json into the user's account-level preferences.
Each top-level key/value pair is upserted as an AccountPreference; unreadable data or a
non-object payload is skipped rather than aborting the whole import.
"""
import json
from authentication.models import AccountPreference
try:
settings = json.loads(data.decode('utf-8'))
if not isinstance(settings, dict):
return 0
imported = 0
for key, value in settings.items():
try:
AccountPreference.objects.update_or_create(user=user, key=key, defaults={'value': value})
imported += 1
except Exception as error:
print(f'Skipping setting "{key}": {error}')
return imported
except Exception as error:
print(f'Skipping settings.json: {error}')
return 0
def delete_user_data(user):
"""Permanently delete everything that `user_data()` exports, keeping the account itself intact.
This removes the user's inventory items (hard delete, bypassing soft-delete), storage
locations, account preferences, the friends relation on their public identity, the profile
picture, and any File blobs (profile picture / inventory attachments) that would otherwise be
orphaned - i.e. not referenced by any other inventory item or user. Files still referenced
elsewhere (they're deduplicated by content hash) are left untouched. The ToolshedUser account
(and its underlying KnownIdentity) is *not* deleted - this only wipes the account's data, it
doesn't close the account. See `delete_account()` in toolshed/api/offlinedata.py for that.
Returns a summary dict describing what was removed.
"""
from django.db import transaction
from toolshed.models import InventoryItem, StorageLocation
summary = {'inventory_items': 0, 'locations': 0, 'settings': 0, 'friends': 0, 'files': 0}
with transaction.atomic():
candidate_file_ids = set(
InventoryItem.global_objects.filter(owner=user).values_list('files__id', flat=True))
candidate_file_ids.discard(None)
if user.profile_picture_id:
candidate_file_ids.add(user.profile_picture_id)
for item in InventoryItem.global_objects.filter(owner=user):
item.hard_delete()
summary['inventory_items'] += 1
summary['locations'], _ = StorageLocation.objects.filter(owner=user).delete()
summary['settings'], _ = user.preferences.all().delete()
summary['friends'] = user.public_identity.friends.count()
user.public_identity.friends.clear()
user.profile_picture = None
user.save(update_fields=['profile_picture'])
summary['files'] = _delete_orphaned_files(candidate_file_ids)
return summary
def delete_user_account(user):
"""Permanently delete the local user's account, after wiping all of its data.
This first calls `delete_user_data()` to remove everything covered by the data export
(inventory, locations, settings, friends relation, profile picture and orphaned files),
then deletes the ToolshedUser row itself. The underlying KnownIdentity is kept so remote
friends/history relating to this identity remain intact - only the local account is closed.
Returns a summary dict describing what was removed, with `account` set to True.
"""
from django.db import transaction
with transaction.atomic():
summary = delete_user_data(user)
user.delete()
summary['account'] = True
return summary
def _delete_orphaned_files(file_ids):
"""Delete File rows (and their underlying blobs) in `file_ids` that are no longer referenced.
A File is considered orphaned once no InventoryItem, no ToolshedUser (profile picture), and no
WorkflowInstance (staged file) references it anymore. Returns the number of files deleted.
"""
from files.models import File
deleted = 0
for file_obj in File.objects.filter(id__in=file_ids):
if file_obj.connected_items.exists() or file_obj.profile_picture_users.exists() \
or file_obj.staged_by_workflows.exists():
continue
file_obj.file.delete(save=False)
file_obj.delete()
deleted += 1
return deleted
def rows_to_csv(rows, fieldnames=None, encoding='utf-8'):
"""Generator that consumes an iterable of dicts and yields encoded CSV data chunk by chunk."""
import csv
class _Echo:
"""A file-like object whose write() just returns what was passed, for streaming csv.writer output."""
def write(self, value):
return value
rows = iter(rows)
try:
first_row = next(rows)
except StopIteration:
return
if fieldnames is None:
fieldnames = list(first_row.keys())
writer = csv.DictWriter(_Echo(), fieldnames=fieldnames)
yield writer.writeheader().encode(encoding)
yield writer.writerow(first_row).encode(encoding)
for row in rows:
yield writer.writerow(row).encode(encoding)
def _read_csv_rows(data, encoding='utf-8'):
"""Decode CSV bytes and yield rows as dicts keyed by the header labels (not column offsets)."""
import csv
import io
reader = csv.DictReader(io.StringIO(data.decode(encoding)))
for row in reader:
yield row
def _field_is_optional(model, field_name):
"""Return True if a field may be omitted: it's a relation, has a default, or allows null/blank."""
try:
field = model._meta.get_field(field_name)
except Exception:
return True
if getattr(field, 'many_to_many', False) or getattr(field, 'one_to_many', False):
return True
return bool(getattr(field, 'null', False) or getattr(field, 'blank', False) or field.has_default())
def get_or_create_category(path):
"""Resolve or create a Category from a '/'-separated path such as 'Electronics/Cables'."""
from toolshed.models import Category
parent = None
category = None
for part in (p for p in path.split('/') if p):
category, _ = Category.objects.get_or_create(name=part, parent=parent, defaults={'origin': 'import'})
parent = category
return category
def import_locations(user, data):
"""Fault-tolerant import of locations.csv into StorageLocations owned by `user`.
Rows are read by header label. Rows missing the required 'name' column are skipped.
Optional columns ('description', 'category') are simply omitted if absent. Locations are
processed in path-depth order so a child's parent already exists by the time it's needed.
Each row runs in its own transaction savepoint so a DB-level failure on one row (e.g. a
constraint violation) can't poison the surrounding transaction and silently break every
subsequent row.
"""
from django.db import transaction
from toolshed.models import StorageLocation
rows = list(_read_csv_rows(data))
rows.sort(key=lambda row: (row.get('path') or row.get('name') or '').count('/'))
resolved_by_path = {}
imported = 0
for row in rows:
try:
with transaction.atomic():
name = (row.get('name') or '').strip()
if not name and not _field_is_optional(StorageLocation, 'name'):
continue # required column missing, skip this row
path = (row.get('path') or name).strip()
parent = None
if '/' in path:
parent = resolved_by_path.get(path.rsplit('/', 1)[0])
category = None
category_path = row.get('category')
if category_path:
category = get_or_create_category(category_path)
location, _ = StorageLocation.objects.update_or_create(
owner=user, name=name, parent=parent,
defaults={
'description': row.get('description', '') or '',
'category': category,
},
)
resolved_by_path[path] = location
imported += 1
except Exception as error:
print(f'Skipping location row {row}: {error}')
return imported
def import_friends(user, data):
"""Fault-tolerant import of friends.csv, adding valid entries to the user's known friends."""
from django.db import transaction
from authentication.models import KnownIdentity
imported = 0
for row in _read_csv_rows(data):
try:
with transaction.atomic():
username = row.get('username')
domain = row.get('domain')
handle = row.get('handle')
if (not username or not domain) and handle and '@' in handle:
username, domain = handle.split('@', 1)
public_key = row.get('public_key')
if not username or not domain or not public_key:
continue # required fields missing, skip this row
identity, _ = KnownIdentity.objects.get_or_create(
username=username, domain=domain,
defaults={'public_key': public_key},
)
user.public_identity.friends.add(identity)
imported += 1
except Exception as error:
print(f'Skipping friend row {row}: {error}')
return imported
class _HandleNotFound(Exception):
"""Raised when a fully qualified handle (e.g. 'git:base#tag:drill') can't be resolved.
This intentionally aborts the whole row (rather than falling back to creating a new
entity), since a handle references a *specific* entity from a *specific* origin - silently
creating a new local entity named after the raw handle string would be incorrect.
"""
def _resolve_handle(value, entity_type, model):
"""Resolve a fully qualified handle (e.g. 'git:base#tag:drill') to an *existing* model instance.
Never creates anything: a handle references a specific entity from a specific origin, so
silently creating a new local entity named after the raw handle string would be incorrect.
Raises `_HandleNotFound` if the handle's entity type doesn't match or no such object exists,
so the caller can skip the row and report a helpful error instead.
"""
origin, rest = value.split('#', 1)
if ':' in rest:
found_type, name = rest.split(':', 1)
if found_type != entity_type:
raise _HandleNotFound(
f"expected a {entity_type} handle but got '{value}' (type '{found_type}')")
else:
name = rest
try:
return model.objects.get(origin=origin, name=name)
except model.DoesNotExist:
raise _HandleNotFound(f"{entity_type} '{value}' does not exist, skipping item")
def _quote_value_if_needed(value):
"""Wrap `value` in double quotes (CSV-style, doubling any embedded quotes) if it contains a
comma or a quote character, so it survives sitting inside a comma-separated "handle=value"
list unambiguously.
"""
if any(ch in value for ch in ',"'):
return '"' + value.replace('"', '""') + '"'
return value
def _encode_properties_cell(item_properties):
"""Encode an item's properties as a comma-separated "handle=value" list for the 'properties'
CSV cell, quoting a value (CSV-style) when it contains a comma or a quote character so it
still round-trips correctly. See `_parse_properties_cell()` for the reader side.
"""
entries = [
f"{ip.property.get_handle()}={_quote_value_if_needed(ip.value or '')}"
for ip in item_properties
]
return ', '.join(entries)
def _split_quoted_comma_list(raw_value):
"""Split a comma-separated list into entries, honouring double-quoted substrings (CSV-style,
wherever they appear in an entry) so a quoted value's own commas aren't mistaken for
separators. Doubled quotes ("") inside a quoted substring are unescaped to a single literal
quote, and the surrounding quotes themselves are stripped from the result.
Only the single space after each ", " separator (as written by `_encode_properties_cell()`)
is dropped - unlike a blanket `.strip()`, this preserves any leading/trailing whitespace that
is genuinely part of a (quoted) value.
"""
entries = []
current = []
in_quotes = False
i = 0
length = len(raw_value)
while i < length:
char = raw_value[i]
if char == '"':
if in_quotes and i + 1 < length and raw_value[i + 1] == '"':
current.append('"')
i += 2
continue
in_quotes = not in_quotes
i += 1
continue
if char == ',' and not in_quotes:
entries.append(''.join(current))
current = []
i += 1
if i < length and raw_value[i] == ' ':
i += 1 # skip the single space of the ", " separator
continue
current.append(char)
i += 1
entries.append(''.join(current))
return [entry for entry in entries if entry]
def _parse_properties_cell(raw_value, resolve_property):
"""Parse the 'properties' CSV cell into a list of (Property, value) tuples.
The cell is a comma-separated list of "handle=value" entries; a value containing a comma or
a quote character is double-quoted (CSV-style) by `_encode_properties_cell()` so it survives
the round trip intact.
"""
raw_value = (raw_value or '').strip()
if not raw_value:
return []
properties = []
for prop_entry in _split_quoted_comma_list(raw_value):
if '=' not in prop_entry:
continue
prop_name, value = prop_entry.split('=', 1)
prop = resolve_property(prop_name.strip())
if prop:
properties.append((prop, value))
return properties
def import_inventory(user, data, available_files):
"""Fault-tolerant import of inventory.csv into InventoryItems owned by `user`.
`available_files` maps the 'files' column's paths to File instances successfully extracted
from the zip; references to missing files are ignored rather than failing the row.
If a row references a fully qualified tag/property/category handle that doesn't exist
locally, the whole item is skipped (rather than creating a bogus local entity named after
the raw handle) and a helpful message is added to the returned `errors` list. Each row runs
in its own transaction savepoint so a DB-level failure on one row can't poison the
surrounding transaction and silently break every subsequent row.
"""
from django.db import transaction
from toolshed.models import Category, InventoryItem, ItemProperty, StorageLocation, Tag, Property
def resolve_category_from_csv(value):
"""Resolve a category from CSV - supports both old format and fully qualified handles"""
if not value:
return None
if '#' in value:
return _resolve_handle(value, 'category', Category)
# Fallback to old path-based lookup
return get_or_create_category(value)
def resolve_tag_from_csv(value):
"""Resolve a tag from CSV - supports both old format and fully qualified handles"""
if not value:
return None
if '#' in value:
return _resolve_handle(value, 'tag', Tag)
# Fallback to old name-only lookup or create
return Tag.objects.get_or_create(name=value, category=None, defaults={'origin': 'import'})[0]
def resolve_property_from_csv(value):
"""Resolve a property from CSV - supports both old format and fully qualified handles"""
if not value:
return None
if '#' in value:
return _resolve_handle(value, 'property', Property)
# Fallback to old name-only lookup or create
return Property.objects.get_or_create(name=value, category=None, defaults={'origin': 'import'})[0]
imported = 0
errors = []
for row in _read_csv_rows(data):
name = (row.get('name') or '').strip()
try:
with transaction.atomic():
file_paths = [p.strip() for p in (row.get('files') or '').split(',') if p.strip()]
files = [available_files[p] for p in file_paths if p in available_files]
if not name and not files:
continue # InventoryItem requires a name or at least one file
category = resolve_category_from_csv((row.get('category') or '').strip())
tags = []
for tag_name in (row.get('tags') or '').split(','):
tag_name = tag_name.strip()
if tag_name:
tag = resolve_tag_from_csv(tag_name)
if tag:
tags.append(tag)
properties = _parse_properties_cell(row.get('properties') or '', resolve_property_from_csv)
storage_location = None
location_path_value = row.get('storage_location')
if location_path_value:
storage_location = StorageLocation.objects.filter(
owner=user, name=location_path_value.rsplit('/', 1)[-1]).first()
try:
owned_quantity = int(row.get('owned_quantity') or 1)
except (TypeError, ValueError):
owned_quantity = 1
item = InventoryItem.create_for_owner(
owner=user,
name=name or None,
description=row.get('description', '') or '',
category=category,
availability_policy=row.get('availability_policy') or 'private',
owned_quantity=owned_quantity,
storage_location=storage_location,
)
for tag in tags:
item.tags.add(tag, through_defaults={})
for prop, value in properties:
ItemProperty.objects.create(inventory_item=item, property=prop, value=value)
for file in files:
item.files.add(file)
imported += 1
except _HandleNotFound as error:
message = f"Skipping item '{name or 'unnamed'}': {error}"
print(message)
errors.append(message)
except Exception as error:
message = f'Skipping inventory row {row}: {error}'
print(message)
errors.append(message)
return imported, errors