add explicit non-interactive option to configure.py

backend/configure.py

@@ -8,32 +8,41 @@ import dotenv
 from django.db import transaction, IntegrityError
 
 
-def yesno(prompt, default=False):
-    if not sys.stdin.isatty():
-        return default
-    yes = {'yes', 'y', 'ye'}
-    no = {'no', 'n'}
+class CmdCtx:
 
-    if default:
-        yes.add('')
-    else:
-        no.add('')
+    def __init__(self, args):
+        self.args = args
 
-    hint = ' [Y/n] ' if default else ' [y/N] '
-
-    while True:
-        choice = input(prompt + hint).lower()
-        if choice in yes:
+    def yesno(self, prompt, default=False):
+        if not sys.stdin.isatty() or self.args.noninteractive:
+            return default
+        elif self.args.yes:
             return True
-        elif choice in no:
+        elif self.args.no:
             return False
+        yes = {'yes', 'y', 'ye'}
+        no = {'no', 'n'}
+
+        if default:
+            yes.add('')
         else:
-            print('Please respond with "yes" or "no"')
+            no.add('')
+
+        hint = ' [Y/n] ' if default else ' [y/N] '
+
+        while True:
+            choice = input(prompt + hint).lower()
+            if choice in yes:
+                return True
+            elif choice in no:
+                return False
+            else:
+                print('Please respond with "yes" or "no"')
 
 
-def configure():
+def configure(ctx):
     if not os.path.exists('.env'):
-        if not yesno("the .env file does not exist, do you want to create it?", default=True):
+        if not ctx.yesno("the .env file does not exist, do you want to create it?", default=True):
             print('Aborting')
             exit(0)
         if not os.path.exists('.env.dist'):
@@ -56,7 +65,7 @@ def configure():
     current_hosts = os.getenv('ALLOWED_HOSTS')
     print('Current ALLOWED_HOSTS: {}'.format(current_hosts))
 
-    if yesno("Do you want to add ALLOWED_HOSTS?"):
+    if ctx.yesno("Do you want to add ALLOWED_HOSTS?"):
         hosts = input("Enter a comma-separated list of allowed hosts: ")
         joined_hosts = current_hosts + ',' + hosts if current_hosts else hosts
         dotenv.set_key('.env', 'ALLOWED_HOSTS', joined_hosts)
@@ -67,20 +76,21 @@ def configure():
     django.setup()
 
     if not os.path.exists('db.sqlite3'):
-        if not yesno("No database found, do you want to create one?", default=True):
+        if not ctx.yesno("No database found, do you want to create one?", default=True):
             print('Aborting')
             exit(0)
 
     from django.core.management import call_command
     call_command('migrate')
 
-    if yesno("Do you want to create a superuser?"):
+    if ctx.yesno("Do you want to create a superuser?"):
         from django.core.management import call_command
         call_command('createsuperuser')
 
     call_command('collectstatic', '--no-input')
 
-    if yesno("Do you want to import all categories, properties and tags contained in this repository?", default=True):
+    if ctx.yesno("Do you want to import all categories, properties and tags contained in this repository?",
+                 default=True):
         from hostadmin.serializers import CategorySerializer, PropertySerializer, TagSerializer
         from hostadmin.models import ImportedIdentifierSets
         from hashlib import sha256
@@ -196,6 +206,7 @@ def main():
     parser = ArgumentParser(description='Toolshed Server Configuration')
     parser.add_argument('--yes', '-y', help='Answer yes to all questions', action='store_true')
     parser.add_argument('--no', '-n', help='Answer no to all questions', action='store_true')
+    parser.add_argument('--noninteractive', '-x', help="Run in noninteractive mode", action='store_true')
     parser.add_argument('cmd', help='Command', default='configure', nargs='?')
     args = parser.parse_args()
 
@@ -203,8 +214,10 @@ def main():
         print('Error: --yes and --no are mutually exclusive')
         exit(1)
 
+    ctx = CmdCtx(args)
+
     if args.cmd == 'configure':
-        configure()
+        configure(ctx)
     elif args.cmd == 'reset':
         reset()
     elif args.cmd == 'testdata':

frontend/src/dns.js

@@ -0,0 +1,50 @@
+import {query} from 'dns-query';
+
+function get_prefered_server() {
+    try {
+        const servers = JSON.parse(localStorage.getItem('dns-servers'));
+        if (servers && servers.length > 0) {
+            return servers;
+        }
+    } catch (e) {
+        console.error(e);
+    }
+    const request = new XMLHttpRequest();
+    request.open('GET', '/local/dns', false);
+    request.send(null);
+    if (request.status === 200) {
+        const servers = JSON.parse(request.responseText);
+        if (servers && servers.length > 0) {
+            return servers;
+        }
+    }
+    return ['1.1.1.1', '8.8.8.8'];
+}
+
+class FallBackResolver {
+    constructor() {
+        this._servers = get_prefered_server();
+        this._cache = JSON.parse(localStorage.getItem('dns-cache')) || {};
+    }
+
+    async query(domain, type) {
+        const key = domain + ':' + type;
+        if (key in this._cache && this._cache[key].time > Date.now() - 1000 * 60 * 60) {
+            const age_seconds = Math.ceil(Date.now() / 1000 - this._cache[key].time / 1000);
+            return [this._cache[key].data];
+        }
+        const result = await query(
+            {question: {type: type, name: domain}},
+            {
+                endpoints: this._servers,
+            }
+        )
+        if (result.answers.length === 0) throw new Error('No answer');
+        const first = result.answers[0];
+        this._cache[key] = {time: Date.now(), ...first}; // TODO hadle multiple answers
+        localStorage.setItem('dns-cache', JSON.stringify(this._cache));
+        return [first.data];
+    }
+}
+
+export default FallBackResolver;

frontend/src/federation.js

@@ -0,0 +1,324 @@
+class ServerSet {
+    constructor(servers, unreachable_neighbors) {
+        if (!servers || !Array.isArray(servers)) {
+            throw new Error('no servers')
+        }
+        if (!unreachable_neighbors || typeof unreachable_neighbors.queryUnreachable !== 'function' || typeof unreachable_neighbors.unreachable !== 'function') {
+            throw new Error('no unreachable_neighbors')
+        }
+        this.servers = [...new Set(servers)] // deduplicate
+        this.unreachable_neighbors = unreachable_neighbors;
+    }
+
+    add(server) {
+        console.log('adding server', server)
+        if (!server || typeof server !== 'string') {
+            throw new Error('server must be a string')
+        }
+        if (server in this.servers) {
+            console.log('server already in set', server)
+            return
+        }
+        this.servers.push(server);
+    }
+
+    async get(auth, target) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'GET',
+                    headers: {
+                        ...auth.buildAuthHeader(url)
+                    },
+                    credentials: 'omit'
+                }).catch(err => {
+                        console.error('get from server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                ).then(response => response.json())
+            } catch (e) {
+                console.error('get from server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+
+    async post(auth, target, data) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        ...auth.buildAuthHeader(url, data)
+                    },
+                    credentials: 'omit',
+                    body: JSON.stringify(data)
+                }).catch(err => {
+                        console.error('post to server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                ).then(response => response.json())
+            } catch (e) {
+                console.error('post to server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+
+    async patch(auth, target, data) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'PATCH',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        ...auth.buildAuthHeader(url, data)
+                    },
+                    credentials: 'omit',
+                    body: JSON.stringify(data)
+                }).catch(err => {
+                        console.error('patch to server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                ).then(response => response.json())
+            } catch (e) {
+                console.error('patch to server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+
+    async put(auth, target, data) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'PUT',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        ...auth.buildAuthHeader(url, data)
+                    },
+                    credentials: 'omit',
+                    body: JSON.stringify(data)
+                }).catch(err => {
+                        console.error('put to server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                ).then(response => response.json())
+            } catch (e) {
+                console.error('put to server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+
+    async delete(auth, target) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'DELETE',
+                    headers: {
+                        ...auth.buildAuthHeader(url)
+                    },
+                    credentials: 'omit'
+                }).catch(err => {
+                        console.error('delete from server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                )
+            } catch (e) {
+                console.error('delete from server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+
+    async getRaw(auth, target) {
+        if (!auth || typeof auth.buildAuthHeader !== 'function') {
+            throw new Error('no auth')
+        }
+        for (const server of this.servers) {
+            try {
+                if (this.unreachable_neighbors.queryUnreachable(server)) {
+                    continue
+                }
+                const url = "https://" + server + target // TODO https
+                return await fetch(url, {
+                    method: 'GET',
+                    headers: {
+                        ...auth.buildAuthHeader(url)
+                    },
+                    credentials: 'omit'
+                }).catch(err => {
+                        console.error('get from server failed', server, err)
+                        this.unreachable_neighbors.unreachable(server)
+                    }
+                )
+            } catch (e) {
+                console.error('get from server failed', server, e)
+            }
+        }
+        throw new Error('all servers failed')
+    }
+}
+
+class ServerSetUnion {
+    constructor(serverSets) {
+        if (!serverSets || !Array.isArray(serverSets)) {
+            throw new Error('no serverSets')
+        }
+        this.serverSets = serverSets;
+    }
+
+    add(serverset) {
+        if (!serverset || !(serverset instanceof ServerSet)) {
+            throw new Error('no serverset')
+        }
+        if (this.serverSets.find(s => serverset.servers.every(s2 => s.servers.includes(s2)))) {
+            console.warn('serverset already in union', serverset)
+            return
+        }
+        this.serverSets.push(serverset)
+    }
+
+    async get(auth, target) {
+        try {
+            return await this.serverSets.reduce(async (acc, serverset) => {
+                return acc.then(async (acc) => {
+                    return acc.concat(await serverset.get(auth, target))
+                })
+            }, Promise.resolve([]))
+        } catch (e) {
+            throw new Error('all servers failed')
+        }
+    }
+
+    async post(auth, target, data) {
+        try {
+            return await this.serverSets.reduce(async (acc, serverset) => {
+                return acc.then(async (acc) => {
+                    return acc.concat(await serverset.post(auth, target, data))
+                })
+            }, Promise.resolve([]))
+        } catch (e) {
+            throw new Error('all servers failed')
+        }
+    }
+
+    async patch(auth, target, data) {
+        try {
+            return await this.serverSets.reduce(async (acc, serverset) => {
+                return acc.then(async (acc) => {
+                    return acc.concat(await serverset.patch(auth, target, data))
+                })
+            }, Promise.resolve([]))
+        } catch (e) {
+            throw new Error('all servers failed')
+        }
+    }
+
+    async put(auth, target, data) {
+        try {
+            return await this.serverSets.reduce(async (acc, serverset) => {
+                return acc.then(async (acc) => {
+                    return acc.concat(await serverset.put(auth, target, data))
+                })
+            }, Promise.resolve([]))
+        } catch (e) {
+            throw new Error('all servers failed')
+        }
+    }
+
+    async delete(auth, target) {
+        try {
+            return await this.serverSets.reduce(async (acc, serverset) => {
+                return acc.then(async (acc) => {
+                    return acc.concat(await serverset.delete(auth, target))
+                })
+            }, Promise.resolve([]))
+        } catch (e) {
+            throw new Error('all servers failed')
+        }
+    }
+}
+
+
+class authMethod {
+    constructor(method, auth) {
+        this.method = method;
+        this.auth = auth;
+    }
+
+    buildAuthHeader(url, data) {
+        return this.method(this.auth, {url, data})
+    }
+
+}
+
+function createSignAuth(username, signKey) {
+    const context = {username, signKey}
+    if (!context.signKey || !context.username || typeof context.username !== 'string'
+        || !(context.signKey instanceof Uint8Array) || context.signKey.length !== 64) {
+        throw new Error('no signKey or username')
+    }
+    return new authMethod(({signKey, username}, {url, data}) => {
+        const json = JSON.stringify(data)
+        const signature = nacl.crypto_sign_detached(nacl.encode_utf8(url + (data ? json : "")), signKey)
+        return {'Authorization': 'Signature ' + username + ':' + nacl.to_hex(signature)}
+    }, context)
+}
+
+function createTokenAuth(token) {
+    const context = {token}
+    if (!context.token) {
+        throw new Error('no token')
+    }
+    return new authMethod(({token}, {url, data}) => {
+        return {'Authorization': 'Token ' + token}
+    }, context)
+}
+
+function createNullAuth() {
+    return new authMethod(() => {
+        return {}
+    }, {})
+}
+
+export {ServerSet, ServerSetUnion, createSignAuth, createTokenAuth, createNullAuth};
+
+

frontend/src/main.js

@@ -5,10 +5,11 @@ import App from './App.vue'
 import './scss/toolshed.scss'
 
 import router from './router'
+import store from './store';
 
 import _nacl from 'js-nacl';
 
-const app = createApp(App).use(BootstrapIconsPlugin);
+const app = createApp(App).use(store).use(BootstrapIconsPlugin);
 
 _nacl.instantiate((nacl) => {
     window.nacl = nacl

frontend/src/neigbors.js

@@ -0,0 +1,48 @@
+class NeighborsCache {
+    constructor() {
+        //this._max_age = 1000 * 60 * 60; // 1 hour
+        //this._max_age = 1000 * 60 * 5; // 5 minutes
+        this._max_age = 1000 * 15; // 15 seconds
+        this._cache = JSON.parse(localStorage.getItem('neighbor-cache')) || {};
+    }
+
+    reachable(domain) {
+        console.log('reachable neighbor ' + domain)
+        if (domain in this._cache) {
+            delete this._cache[domain];
+            localStorage.setItem('neighbor-cache', JSON.stringify(this._cache));
+        }
+    }
+
+    unreachable(domain) {
+        console.log('unreachable neighbor ' + domain)
+        this._cache[domain] = {time: Date.now()};
+        localStorage.setItem('neighbor-cache', JSON.stringify(this._cache));
+    }
+
+    queryUnreachable(domain) {
+        //return false if unreachable
+        if (domain in this._cache) {
+            if (this._cache[domain].time > Date.now() - this._max_age) {
+                console.log('skip unreachable neighbor ' + domain + ' ' + Math.ceil(
+                    Date.now()/1000 - this._cache[domain].time/1000) + 's/' + Math.ceil(this._max_age/1000) + 's')
+                return true
+            } else {
+                delete this._cache[domain];
+                localStorage.setItem('neighbor-cache', JSON.stringify(this._cache));
+            }
+        }
+        return false;
+    }
+
+    list() {
+        return Object.entries(this._cache).map(([domain, elem]) => {
+            return {
+                domain: domain,
+                time: elem.time
+            }
+        })
+    }
+}
+
+export default NeighborsCache;

frontend/src/store.js

@@ -0,0 +1,132 @@
+import {createStore} from 'vuex';
+import router from '@/router';
+import FallBackResolver from "@/dns";
+import NeighborsCache from "@/neigbors";
+import {createNullAuth, createSignAuth, createTokenAuth, ServerSet, ServerSetUnion} from "@/federation";
+
+
+export default createStore({
+    state: {
+        local_loaded: false,
+        last_load: {},
+        user: null,
+        token: null,
+        keypair: null,
+        remember: false,
+        home_servers: null,
+        resolver: new FallBackResolver(),
+        unreachable_neighbors: new NeighborsCache(),
+    },
+    mutations: {
+        setUser(state, user) {
+            state.user = user;
+            if (state.remember)
+                localStorage.setItem('user', user);
+        },
+        setToken(state, token) {
+            state.token = token;
+            if (state.remember)
+                localStorage.setItem('token', token);
+        },
+        setKey(state, keypair) {
+            state.keypair = nacl.crypto_sign_keypair_from_seed(nacl.from_hex(keypair))
+            if (state.remember)
+                localStorage.setItem('keypair', nacl.to_hex(state.keypair.signSk).slice(0, 64))
+        },
+        setRemember(state, remember) {
+            state.remember = remember;
+            if (!remember) {
+                localStorage.removeItem('user');
+                localStorage.removeItem('token');
+                localStorage.removeItem('keypair');
+            }
+            localStorage.setItem('remember', remember);
+        },
+        setHomeServers(state, home_servers) {
+            state.home_servers = home_servers;
+        },
+        logout(state) {
+            state.user = null;
+            state.token = null;
+            state.keypair = null;
+            localStorage.removeItem('user');
+            localStorage.removeItem('token');
+            localStorage.removeItem('keypair');
+            router.push('/login');
+        },
+        load_local(state) {
+            if (state.local_loaded)
+                return;
+            const remember = localStorage.getItem('remember');
+            const user = localStorage.getItem('user');
+            const token = localStorage.getItem('token');
+            const keypair = localStorage.getItem('keypair');
+            if (user && token) {
+                this.commit('setUser', user);
+                this.commit('setToken', token);
+                if (keypair) {
+                    this.commit('setKey', keypair)
+                }
+            }
+            state.cache_loaded = true;
+        }
+    },
+    actions: {
+        async login({commit, dispatch, state, getters}, {username, password, remember}) {
+            commit('setRemember', remember);
+            const data = await dispatch('lookupServer', {username}).then(servers => new ServerSet(servers, state.unreachable_neighbors))
+                .then(set => set.post(getters.nullAuth, '/auth/token/', {username, password}))
+            if (data.token && data.key) {
+                commit('setToken', data.token);
+                commit('setUser', username);
+                commit('setKey', data.key);
+                const s = await dispatch('lookupServer', {username}).then(servers => new ServerSet(servers, state.unreachable_neighbors))
+                commit('setHomeServers', s)
+                return true;
+            } else {
+                return false;
+            }
+        },
+        async lookupServer({state}, {username}) {
+            const domain = username.split('@')[1]
+            const request = '_toolshed-server._tcp.' + domain + '.'
+            return await state.resolver.query(request, 'SRV').then(
+                (result) => result.map(
+                    (answer) => answer.target + ':' + answer.port))
+        },
+        async getHomeServers({state, dispatch, commit}) {
+            if (state.home_servers)
+                return state.home_servers
+            const promise = dispatch('lookupServer', {username: state.user}).then(servers => new ServerSet(servers, state.unreachable_neighbors))
+            commit('setHomeServers', promise)
+            return promise
+        },
+        async getFriendServers({state, dispatch, commit}, {username}) {
+            return dispatch('lookupServer', {username}).then(servers => new ServerSet(servers, state.unreachable_neighbors))
+        },
+    },
+    getters: {
+        isLoggedIn(state) {
+            if (!state.local_loaded) {
+                state.remember = localStorage.getItem('remember') === 'true'
+                state.user = localStorage.getItem('user')
+                state.token = localStorage.getItem('token')
+                const keypair = localStorage.getItem('keypair')
+                if (keypair)
+                    state.keypair = nacl.crypto_sign_keypair_from_seed(nacl.from_hex(keypair))
+                state.local_loaded = true
+            }
+
+            return state.user !== null && state.token !== null;
+        },
+        signAuth(state) {
+            return createSignAuth(state.user, state.keypair.signSk)
+        },
+        tokenAuth(state) {
+            return createTokenAuth(state.token)
+        },
+        nullAuth(state) {
+            return createNullAuth({})
+        },
+    }
+})