move ssb-patchwork-api and ssb-patchwork-ui into this repo

.gitignore

@@ -1,4 +1,4 @@
 dist/
 node_modules/
 npm-debug.log
-
+ui/css/*

api/index.js

@@ -0,0 +1,508 @@
+var fs          = require('fs')
+var pull        = require('pull-stream')
+var multicb     = require('multicb')
+var pl          = require('pull-level')
+var pushable    = require('pull-pushable')
+var paramap     = require('pull-paramap')
+var cat         = require('pull-cat')
+var Notify      = require('pull-notify')
+var toPull      = require('stream-to-pull-stream')
+var ref         = require('ssb-ref')
+var pathlib     = require('path')
+var u           = require('./util')
+
+exports.name        = 'patchwork'
+exports.version     = '1.0.0'
+exports.manifest    = require('./manifest')
+exports.permissions = require('./permissions')
+
+exports.init = function (sbot, opts) {
+
+  var api = {}
+  var phoenixdb = sbot.sublevel('patchwork')
+  var db = {
+    isread: phoenixdb.sublevel('isread'),
+    subscribed: phoenixdb.sublevel('subscribed')
+  }
+  var state = {
+    // indexes (lists of {key:, ts:})
+    mymsgs: [],
+    home: u.index(), // also has `.isread`
+    inbox: u.index(), // also has `.isread` and `.author`
+    votes: u.index(), // also has `.isread`, `.vote`, and `.votemsg`
+    myvotes: u.index(), // also has  `.vote`
+    follows: u.index(), // also has `.isread` and `.following`
+
+    // views
+    profiles: {},
+    sites: {},
+    names: {}, // ids -> names
+    ids: {}, // names -> ids
+    actionItems: {}
+  }
+
+  var processor = require('./processor')(sbot, db, state, emit)
+  pull(pl.read(sbot.sublevel('log'), { live: true, onSync: onPrehistorySync }), pull.drain(processor))
+
+  // track sync state
+  // - processor does async processing for each message that comes in
+  // - awaitSync() waits for that processing to finish
+  // - pinc() on message arrival, pdec() on message processed
+  // - nP === 0 => all messages processed
+  var nP = 0, syncCbs = []
+  function awaitSync (cb) {
+    if (nP > 0)
+      syncCbs.push(cb)
+    else cb()
+  }
+  state.pinc = function () { nP++ }
+  state.pdec = function () {
+    nP--
+    if (nP === 0) {
+      syncCbs.forEach(function (cb) { cb() })
+      syncCbs.length = 0
+    }
+  }
+
+  var isPreHistorySynced = false // track so we dont emit events for old messages
+  // grab for history sync
+  state.pinc()
+  function onPrehistorySync () {
+    console.log('Log history read...')
+    // when all current items finish, consider prehistory synced (and start emitting)
+    awaitSync(function () { 
+      console.log('Indexes generated')
+      isPreHistorySynced = true
+    })
+    // release
+    state.pdec()
+  }
+
+  // events stream
+  var notify = Notify()
+  function emit (type, data) {
+    if (!isPreHistorySynced)
+      return
+    var e = data || {}
+    e.type = type
+    if (e.type == 'index-change') {
+      api.getIndexCounts(function (err, counts) {
+        e.total = counts[e.index]
+        e.unread = counts[e.index+'Unread']
+        notify(e)
+      })
+    } else
+      notify(e)
+  }
+
+  // getters
+
+  api.createEventStream = function () {
+    return notify.listen()
+  }
+
+  api.getPaths = function (cb) {
+    cb(null, {
+      site: pathlib.join(opts.path, 'publish')
+    })
+  }
+
+  api.getMyProfile = function (cb) {
+    awaitSync(function () {
+      api.getProfile(sbot.id, cb)
+    })
+  }
+
+  function isInboxFriend (row) {
+    if (row.author == sbot.id) return true
+    var p = state.profiles[sbot.id]
+    if (!p) return false
+    return p.assignedTo[row.author] && p.assignedTo[row.author].following
+  }
+
+  api.getIndexCounts = function (cb) {
+    awaitSync(function () {
+      cb(null, {
+        inbox: state.inbox.rows.filter(isInboxFriend).length,
+        inboxUnread: state.inbox.filter(function (row) { return isInboxFriend(row) && row.author != sbot.id && !row.isread }).length,
+        votes: state.votes.filter(function (row) { return row.vote > 0 }).length,
+        votesUnread: state.votes.filter(function (row) { return row.vote > 0 && !row.isread }).length,
+        follows: state.follows.filter(function (row) { return row.following }).length,
+        followsUnread: state.follows.filter(function (row) { return row.following && !row.isread }).length,
+        home: state.home.rows.length
+      })
+    })
+  }
+
+  api.createInboxStream = indexStreamFn(state.inbox, function (row) { 
+    if (!isInboxFriend(row)) return false
+    return row.key
+  })
+  api.createVoteStream = indexStreamFn(state.votes, function (row) { 
+    if (row.vote <= 0) return false
+    return row.votemsg
+  })
+  api.createMyvoteStream = indexStreamFn(state.myvotes, function (row) { 
+    if (row.vote <= 0) return false
+    return row.key
+  })
+  api.createFollowStream = indexStreamFn(state.follows)
+  api.createHomeStream = indexStreamFn(state.home)
+
+  function indexMarkRead (indexname, key, keyname) {
+    if (Array.isArray(key)) {
+      key.forEach(function (k) {
+        indexMarkRead(indexname, k, keyname)
+      })
+      return
+    }
+
+    var index = state[indexname]
+    var row = index.find(key, keyname)
+    if (row) {
+      var wasread = row.isread
+      row.isread = true
+      if (!wasread)
+        emit('index-change', { index: indexname })
+    }
+  }
+
+  function indexMarkUnread (indexname, key, keyname) {
+    if (Array.isArray(key)) {
+      key.forEach(function (k) {
+        indexMarkUnread(indexname, k, keyname)
+      })
+      return
+    }
+
+    var index = state[indexname]
+    var row = index.find(key, keyname)
+    if (row) {
+      var wasread = row.isread
+      row.isread = false
+      if (wasread)
+        emit('index-change', { index: indexname })
+    }
+  }
+
+  api.markRead = function (key, cb) {
+    indexMarkRead('inbox', key)
+    indexMarkRead('votes', key, 'votemsg')
+    indexMarkRead('follows', key)
+    if (Array.isArray(key))
+      db.isread.batch(key.map(function (k) { return { type: 'put', key: k, value: 1 }}), cb)
+    else
+      db.isread.put(key, 1, cb)
+  }
+  api.markUnread = function (key, cb) {
+    indexMarkUnread('inbox', key)
+    indexMarkUnread('votes', key, 'votemsg')
+    indexMarkUnread('follows', key)
+    if (Array.isArray(key))
+      db.isread.batch(key.map(function (k) { return { type: 'del', key: k }}), cb)
+    else
+      db.isread.del(key, cb) 
+  }
+  api.toggleRead = function (key, cb) {
+    api.isRead(key, function (err, v) {
+      if (!v) {
+        api.markRead(key, function (err) {
+          cb(err, true)
+        })
+      } else {
+        api.markUnread(key, function (err) {
+          cb(err, false)
+        })
+      }
+    })
+  }
+  api.isRead = function (key, cb) {
+    if (Array.isArray(key)) {
+      var done = multicb({ pluck: 1 })
+      key.forEach(function (k, i) {
+        var cb = done()
+        db.isread.get(k, function (err, v) { cb(null, !!v) })
+      })
+      done(cb)
+    } else {
+      db.isread.get(key, function (err, v) {
+        cb && cb(null, !!v)
+      })
+    }
+  }
+ 
+  api.subscribe = function (key, cb) {
+    db.subscribed.put(key, 1, cb)
+  }
+  api.unsubscribe = function (key, cb) {
+    db.subscribed.del(key, cb) 
+  }
+  api.toggleSubscribed = function (key, cb) {
+    api.isSubscribed(key, function (err, v) {
+      if (!v) {
+        api.subscribe(key, function (err) {
+          cb(err, true)
+        })
+      } else {
+        api.unsubscribe(key, function (err) {
+          cb(err, false)
+        })
+      }
+    })
+  }
+  api.isSubscribed = function (key, cb) {
+    db.subscribed.get(key, function (err, v) {
+      cb && cb(null, !!v)
+    })
+  }
+
+  api.addFileToBlobs = function (path, cb) {
+    pull(
+      toPull.source(fs.createReadStream(path)),
+      sbot.blobs.add(function (err, hash) {
+        if (err)
+          cb(err)
+        else {
+          var ext = pathlib.extname(path)
+          if (ext == '.png' || ext == '.jpg' || ext == '.jpeg') {
+            var res = getImgDim(path)
+            res.hash = hash
+            cb(null, res)
+          } else
+            cb(null, { hash: hash })
+        }
+      })
+    )
+  }
+  api.saveBlobToFile = function (hash, path, cb) {
+    pull(
+      sbot.blobs.get(hash),
+      toPull.sink(fs.createWriteStream(path), cb)
+    )
+  }
+  function getImgDim (path) {
+    var NativeImage = require('native-image')
+    var ni = NativeImage.createFromPath(path)
+    return ni.getSize()
+  }
+
+  var lookupcodeRegex = /(@[a-z0-9\/\+\=]+\.[a-z0-9]+)(?:\[via\])?(.+)?/i
+  api.useLookupCode = function (code) {
+    var eventPush = pushable()
+
+    // parse and validate the code
+    var id, addrs
+    var parts = lookupcodeRegex.exec(code)
+    var valid = true
+    if (parts) {
+      id  = parts[1]
+      addrs = (parts[2]) ? parts[2].split(',') : []
+
+      // validate id
+      if (!ref.isFeedId(id))
+        valid = false
+
+      // parse addresses
+      addrs = addrs
+        .map(function (addr) {
+          addr = addr.split(':')
+          if (addr.length === 3)
+            return { host: addr[0], port: +addr[1], key: addr[2] }
+        })
+        .filter(Boolean)
+    } else
+      valid = false
+
+    if (!valid) {
+      eventPush.push({ type: 'error', message: 'Invalid lookup code' })
+      eventPush.end()
+      return eventPush
+    }
+
+    // begin the search!
+    search(addrs.concat(sbot.gossip.peers()))
+    function search (peers) {
+      var peer = peers.pop()
+      if (!peer)
+        return eventPush.end()
+
+      // connect to the peer
+      eventPush.push({ type: 'connecting', addr: peer })      
+      sbot.connect(peer, function (err, rpc) {
+        if (err) {
+          eventPush.push({ type: 'error', message: 'Failed to connect', err: err })
+          return search(peers)
+        }
+        // try a sync
+        sync(rpc, function (err, seq) { 
+          if (seq > 0) {
+            // success!
+            eventPush.push({ type: 'finished', seq: seq })
+            eventPush.end()
+          } else
+            search(peers) // try next
+        })
+      })
+    }
+
+    function sync (rpc, cb) {
+      // fetch the feed
+      var seq
+      eventPush.push({ type: 'syncing', id: id })
+      pull(
+        rpc.createHistoryStream({ id: id, keys: false }),
+        pull.through(function (msg) {
+          seq = msg.sequence
+        }),
+        sbot.createWriteStream(function (err) {
+          cb(err, seq)
+        })
+      )
+    }
+
+    return eventPush
+  }
+
+  api.getSite = function (id, cb) {
+    awaitSync(function () { cb(null, state.sites[id]) })
+  }
+
+  var sitePathRegex = /(@.*\.ed25519)(.*)/
+  api.getSiteLink = function (url, cb) {
+    awaitSync(function () {
+      // parse url
+      var parts = sitePathRegex.exec(url)
+      if (!parts) {
+        var err = new Error('Not found')
+        err.notFound = true
+        return cb(err)
+      }
+
+      var pid = parts[1]
+      var path = parts[2]
+      if (path.charAt(0) == '/')
+        path = path.slice(1) // skip the preceding slash
+      if (!path)
+        path = 'index.html' // default asset
+
+      // lookup the link
+      var link = (state.sites[pid]) ? state.sites[pid][path] : null
+      if (!link) {
+        var err = new Error('Not found')
+        err.notFound = true
+        return cb(err)
+      }
+      cb(null, link)
+    })
+  }
+
+  api.getProfile = function (id, cb) {
+    awaitSync(function () { cb(null, state.profiles[id]) })
+  }
+  api.getAllProfiles = function (cb) {
+    awaitSync(function () { cb(null, state.profiles) })
+  }
+  api.getNamesById = function (cb) {
+    awaitSync(function () { cb(null, state.names) })
+  }
+  api.getName = function (id, cb) {
+    awaitSync(function () { cb(null, state.names[id]) })
+  }
+  api.getIdsByName = function (cb) {
+    awaitSync(function () { cb(null, state.ids) })
+  }
+  api.getActionItems = function (cb) {
+    awaitSync(function () { cb(null, state.actionItems) })
+  }
+
+  // helper to get an option off an opt function (avoids the `opt || {}` pattern)
+  function o (opts, k, def) {
+    return opts && opts[k] !== void 0 ? opts[k] : def
+  }
+
+  // helper to get messages from an index
+  function indexStreamFn (index, getkey) {
+    return function (opts) {
+      // emulate the `ssb.createFeedStream` interface
+      var lt    = o(opts, 'lt')
+      var lte   = o(opts, 'lte')
+      var gt    = o(opts, 'gt')
+      var gte   = o(opts, 'gte')
+      var limit = o(opts, 'limit')
+
+      // lt, lte, gt, gte should look like:
+      // [msg.value.timestamp, msg.value.author]
+
+      // helper to create emittable rows
+      function lookup (row) {
+        if (!row) return
+        var key = (getkey) ? getkey(row) : row.key
+        if (key) {
+          var rowcopy = { key: key }
+          for (var k in row) { // copy index attrs into rowcopy
+            if (!rowcopy[k]) rowcopy[k] = row[k]
+          }
+          return rowcopy
+        }
+      }
+
+      // helper to fetch rows
+      function fetch (row, cb) {
+        sbot.get(row.key, function (err, value) {
+          // if (err) {
+            // suppress this error
+            // the message isnt in the local cache (yet)
+            // but it got into the index, likely due to a link
+            // instead of an error, we'll put a null there to indicate the gap
+          // }
+          row.value = value
+          cb(null, row)
+        })
+      }
+
+      // readstream
+      var readPush = pushable()
+      var read = pull(readPush, paramap(fetch))
+
+      // await sync, then emit the reads
+      awaitSync(function () {
+        var added = 0
+        for (var i=0; i < index.rows.length; i++) {
+          var row = index.rows[i]
+
+          if (limit && added >= limit)
+            break
+
+          // we're going to only look at timestamp, because that's all that phoenix cares about
+          var invalid = !!(
+            (lt  && row.ts >= lt[0]) ||
+            (lte && row.ts > lte[0]) ||
+            (gt  && row.ts <= gt[0]) ||
+            (gte && row.ts < gte[0])
+          )
+          if (invalid)
+            continue
+
+          var r = lookup(row)
+          if (r) {
+            readPush.push(r)
+            added++
+          }
+        }
+        readPush.end()
+      })
+
+      if (opts && opts.live) {
+        // live stream, concat the live-emitter on the end
+        index.on('add', onadd)
+        var livePush = pushable(function () { index.removeListener('add', onadd) })
+        function onadd (row) { livePush.push(lookup(row)) }
+        var live = pull(livePush, paramap(fetch))
+        return cat([read, live])
+      }
+      return read
+    }
+  }
+
+  return api
+}

api/manifest.js

@@ -0,0 +1,38 @@
+module.exports = {
+  createEventStream: 'source',
+  getPaths: 'async',
+
+  getIndexCounts: 'async',
+  createInboxStream: 'source',
+  createVoteStream: 'source',
+  createMyvoteStream: 'source',
+  createFollowStream: 'source',
+  createHomeStream: 'source',
+
+  markRead: 'async',
+  markUnread: 'async',
+  toggleRead: 'async',
+  isRead: 'async',
+
+  subscribe: 'async',
+  unsubscribe: 'async',
+  toggleSubscribed: 'async',
+  isSubscribed: 'async',
+
+  addFileToBlobs: 'async',
+  saveBlobToFile: 'async',
+
+  useLookupCode: 'source',
+
+  getMyProfile: 'async',
+  getProfile: 'async',
+  getAllProfiles: 'async',
+
+  getSite: 'async',
+  getSiteLink: 'async',
+
+  getNamesById: 'async',
+  getName: 'async',
+  getIdsByName: 'async',
+  getActionItems: 'async'
+}

api/permissions.js

@@ -0,0 +1 @@
+module.exports = { anonymous: { allow: [] } } // allow nothing unless master

api/processor.js

@@ -0,0 +1,370 @@
+var mlib = require('ssb-msgs')
+var u = require('./util')
+
+module.exports = function (sbot, db, state, emit) {
+
+  var processors = {
+    post: function (msg) {
+      var me = getProfile(sbot.id)
+      var author = msg.value.author
+      var by_me = (author === sbot.id)
+      var c = msg.value.content
+      
+      // home index
+      if (c.recps)
+        return // skip targeted messages
+
+      state.home.sortedUpsert(msg.value.timestamp, msg.key)
+      if (mlib.link(c.root, 'msg')) {
+        // a reply, put its *parent* in the home index
+        state.pinc()
+        u.getRootMsg(sbot, msg, function (err, rootmsg) {
+          if (rootmsg && typeof rootmsg.value.content != 'string') // dont put encrypted msgs in homestream
+            state.home.sortedUpsert(rootmsg.value.timestamp, rootmsg.key)
+          state.pdec()            
+        })
+      }
+
+      if (!by_me) {
+        // emit home-add if by a followed user and in the last hour
+        if (follows(sbot.id, author) && ((Date.now() - msg.value.timestamp) < 1000*60*60))
+          emit('home-add')
+      }
+
+      // inbox index
+      if (!by_me) {
+        var inboxed = false
+        mlib.links(c.root, 'msg').forEach(function (link) {
+          if (inboxed) return
+          // a reply to my messages?
+          if (state.mymsgs.indexOf(link.link) >= 0) {
+            var row = state.inbox.sortedInsert(msg.value.timestamp, msg.key)
+            attachIsRead(row)
+            row.author = msg.value.author // inbox index is filtered on read by the friends graph
+            if (follows(sbot.id, row.author))
+              emit('index-change', { index: 'inbox' })
+            inboxed = true
+          }
+        })
+        mlib.links(c.mentions, 'feed').forEach(function (link) {
+          if (inboxed) return
+          // mentions me?
+          if (link.link == sbot.id) {
+            var row = state.inbox.sortedInsert(msg.value.timestamp, msg.key)
+            attachIsRead(row)
+            row.author = msg.value.author // inbox index is filtered on read by the friends graph
+            if (follows(sbot.id, row.author))
+              emit('index-change', { index: 'inbox' })
+            inboxed = true
+          }
+        })
+      }
+    },
+
+    site: function (msg) {
+      var site = getSite(msg.value.author)
+
+      // additions
+      mlib.links(msg.value.content.includes, 'blob').forEach(function (link) {
+        if (!link.path)
+          return
+        site[link.path] = link
+      })
+
+      // removals
+      var excludes = msg.value.content.excludes
+      if (excludes) {
+        ;(Array.isArray(excludes) ? excludes : [excludes]).forEach(function (item) {
+          if (!item.path)
+            return
+          delete site[item.path]
+        })
+      }
+    },
+
+    contact: function (msg) {
+      // update profiles
+      mlib.links(msg.value.content.contact, 'feed').forEach(function (link) {
+        var toself = link.link === msg.value.author
+        if (toself) updateSelfContact(msg.value.author, msg)
+        else        updateOtherContact(msg.value.author, link.link, msg)
+      })
+    },
+
+    about: function (msg) {
+      // update profiles
+      mlib.links(msg.value.content.about, 'feed').forEach(function (link) {
+        var toself = link.link === msg.value.author
+        if (toself) updateSelfContact(msg.value.author, msg)
+        else        updateOtherContact(msg.value.author, link.link, msg)
+      })
+    },
+
+    vote: function (msg) {
+      // update tallies
+      var link = mlib.link(msg.value.content.vote, 'msg')
+
+      if (link) {
+        if (msg.value.author == sbot.id)
+          updateMyVote(msg, link)
+        else if (state.mymsgs.indexOf(link.link) >= 0) // vote on my msg?
+          updateVoteOnMymsg(msg, link)
+      }
+    },
+
+    flag: function (msg) {
+      // inbox index
+      var link = mlib.link(msg.value.content.flag, 'msg')
+      if (sbot.id != msg.value.author && link && state.mymsgs.indexOf(link.link) >= 0) {
+        var row = state.inbox.sortedInsert(msg.value.timestamp, msg.key)
+        attachIsRead(row)
+        row.author = msg.value.author // inbox index is filtered on read by the friends graph
+        if (follows(sbot.id, msg.value.author))
+          emit('index-change', { index: 'inbox' })
+      }
+
+      // user flags
+      var link = mlib.link(msg.value.content.flag, 'feed')
+      if (link) {
+        var source = getProfile(msg.value.author)
+        var target = getProfile(link.link)
+
+        var flag = link.reason ? { key: msg.key, reason: link.reason } : false
+        source.assignedTo[target.id].flagged = flag
+        target.assignedBy[source.id].flagged = flag
+
+        // track if by local user
+        if (source.id === sbot.id)
+          target.flagged = flag
+      }
+    }
+  }
+
+  function getProfile (pid) {
+    if (pid.id) // already a profile?
+      return pid
+
+    var profile = state.profiles[pid]
+    if (!profile) {
+      state.profiles[pid] = profile = {
+        id: pid,
+
+        // current values...
+        self: { name: null, image: null }, // ...set by self about self
+        assignedBy: {}, // ...set by others about self
+        assignedTo: {}, // ...set by self about others
+
+        // has local user flagged?
+        flagged: false
+      }
+    }
+    return profile
+  }
+
+  function getSite (pid) {
+    var site = state.sites[pid]
+    if (!site)
+      state.sites[pid] = site = {}
+    return site
+  }
+
+  function updateSelfContact (author, msg) {
+    var c = msg.value.content
+    author = getProfile(author)
+
+    // name: a non-empty string
+    if (nonEmptyStr(c.name)) {
+      author.self.name = makeNameSafe(c.name)
+      rebuildNamesFor(author)
+    }
+
+    // image: link to image
+    if ('image' in c) {
+      if (mlib.link(c.image, 'blob'))
+        author.self.image = mlib.link(c.image)
+      else if (!c.image)
+        delete author.self.image
+    }
+  }
+
+  function updateOtherContact (source, target, msg) {
+    var c = msg.value.content
+    source = getProfile(source)
+    target = getProfile(target)
+    source.assignedTo[target.id] = source.assignedTo[target.id] || {}
+    target.assignedBy[source.id] = target.assignedBy[source.id] || {}
+    var userProf = getProfile(sbot.id)
+
+    // name: a non-empty string
+    if (nonEmptyStr(c.name)) {
+      source.assignedTo[target.id].name = makeNameSafe(c.name)
+      target.assignedBy[source.id].name = makeNameSafe(c.name)
+      rebuildNamesFor(target)
+    }
+
+    // following: bool
+    if (typeof c.following === 'boolean') {
+      source.assignedTo[target.id].following = c.following
+      target.assignedBy[source.id].following = c.following
+
+      // if from the user, update names (in case un/following changes conflict status)
+      if (source.id == sbot.id)
+        rebuildNamesFor(target)
+
+      // follows index
+      if (target.id == sbot.id) {
+        // use the follower's id as the key to this index, so we only have 1 entry per other user max
+        var row = state.follows.sortedUpsert(msg.value.timestamp, msg.key)
+        row.following = c.following
+        attachIsRead(row, msg.key)
+      }
+    }
+
+    // blocking: bool
+    if (typeof c.blocking === 'boolean') {
+      source.assignedTo[target.id].blocking = c.blocking
+      target.assignedBy[source.id].blocking = c.blocking
+    }
+  }
+
+  function rebuildNamesFor (profile) {
+    profile = getProfile(profile)
+
+    // remove oldname from id->name map
+    var oldname = state.names[profile.id]
+    if (oldname) {
+      if (state.ids[oldname] == profile.id) {
+        // remove
+        delete state.ids[oldname]
+      } else if (Array.isArray(state.ids[oldname])) {
+        // is in a conflict, remove from conflict array
+        var i = state.ids[oldname].indexOf(profile.id)
+        if (i !== -1) {
+          state.ids[oldname].splice(i, 1)
+          if (state.ids[oldname].length === 1) {
+            // conflict resolved
+            delete state.actionItems[oldname]
+            state.ids[oldname] = state.ids[oldname][0]
+          }
+        }
+      }
+    }
+
+    // default to self-assigned name
+    var name = profile.self.name
+    if (profile.id !== sbot.id && profile.assignedBy[sbot.id] && profile.assignedBy[sbot.id].name) {
+      // use name assigned by the local user, if one is given
+      name = profile.assignedBy[sbot.id].name
+    }
+    if (!name)
+      return
+
+    // store
+    state.names[profile.id] = name
+
+    // if following, update id->name map
+    if (profile.id === sbot.id || profile.assignedBy[sbot.id] && profile.assignedBy[sbot.id].following) {
+      if (!state.ids[name]) { // no conflict?
+        // take it
+        state.ids[name] = profile.id
+      } else {
+        // keep track of all assigned ids
+        if (Array.isArray(state.ids[name]))
+          state.ids[name].push(profile.id)
+        else
+          state.ids[name] = [state.ids[name], profile.id]
+        // conflict, this needs to be handled by the user
+        state.actionItems[name] = {
+          type: 'name-conflict',
+          name: name,
+          ids: state.ids[name]
+        }
+      }
+    }
+  }
+
+  function updateMyVote (msg, l) {
+    // myvotes index
+    var row = state.myvotes.sortedUpsert(msg.value.timestamp, l.link)
+    row.vote = l.value
+  }
+
+  function updateVoteOnMymsg (msg, l) {
+    // votes index
+    // construct a composite key which will be the same for all votes by this user on the given target
+    var votekey = l.link + '::' + msg.value.author // lonnng fucking key
+    var row = state.votes.sortedUpsert(msg.value.timestamp, votekey)
+    row.vote = l.value
+    row.votemsg = msg.key
+    if (row.vote > 0) attachIsRead(row, msg.key)
+    else              row.isread = true // we dont care about non-upvotes
+  }
+
+  function attachIsRead (indexRow, key) {
+    key = key || indexRow.key
+    state.pinc()
+    db.isread.get(key, function (err, v) {
+      state.pdec()
+      indexRow.isread = !!v
+    })
+  }
+
+  function follows (a, b) {
+    var aT = getProfile(a).assignedTo[b]
+    return (a != b && aT && aT.following)
+  }
+
+  // exported api
+
+  function fn (logkey) {
+    state.pinc()
+    var key = logkey.value
+    sbot.get(logkey.value, function (err, value) {
+      var msg = { key: key, value: value }
+      try {
+        // encrypted? try to decrypt
+        if (typeof value.content == 'string' && value.content.slice(-4) == '.box') {
+          value.content = sbot.private.unbox(value.content)
+          if (!value.content)
+            return state.pdec()
+
+          // put all decrypted messages in the inbox index
+          var row = state.inbox.sortedInsert(msg.value.timestamp, msg.key)
+          attachIsRead(row)
+          row.author = msg.value.author // inbox index is filtered on read by the friends graph
+          if (follows(sbot.id, msg.value.author))
+            emit('index-change', { index: 'inbox' })
+        }
+
+        // collect keys of user's messages
+        if (msg.value.author === sbot.id)
+          state.mymsgs.push(msg.key)
+
+        // type processing
+        var process = processors[msg.value.content.type]
+        if (process)
+          process(msg)
+      }
+      catch (e) {
+        // :TODO: use sbot logging plugin
+        console.error('Failed to process message', e, e.stack, key, value)
+      }
+      state.pdec()
+    })
+  }
+
+  return fn
+}
+
+function nonEmptyStr (str) {
+    return (typeof str === 'string' && !!(''+str).trim())
+  }
+
+// allow A-z0-9._-, dont allow a trailing .
+var badNameCharsRegex = /[^A-z0-9\._-]/g
+function makeNameSafe (str) {
+  str = str.replace(badNameCharsRegex, '_')
+  if (str.charAt(str.length - 1) == '.')
+    str = str.slice(0, -1) + '_'
+  return str
+}

api/test/indexes.js

@@ -0,0 +1,389 @@
+var multicb = require('multicb')
+var tape    = require('tape')
+var ssbkeys = require('ssb-keys')
+var pull    = require('pull-stream')
+var u       = require('./util')
+
+tape('inbox index includes encrypted messages from followeds', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    var done = multicb()
+    users.bob.add(ssbkeys.box({ type: 'post', text: 'hello from bob' }, [users.alice.keys, users.bob.keys]), done())
+    users.charlie.add(ssbkeys.box({ type: 'post', text: 'hello from charlie' }, [users.alice.keys, users.charlie.keys]), done())
+    done(function (err) {
+      if (err) throw err
+
+      pull(sbot.patchwork.createInboxStream(), pull.collect(function (err, msgs) {
+        if (err) throw err
+        t.equal(msgs.length, 1)
+        t.equal(msgs[0].value.author, users.bob.id)
+        t.end()
+        sbot.close()
+      }))
+    })
+  })
+})
+
+tape('inbox index includes replies to the users posts from followeds', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'post', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'post', text: 'hello from bob', root: msg.key, branch: msg.key }, done())
+      users.charlie.add({ type: 'post', text: 'hello from charlie', root: msg.key, branch: msg.key }, done())
+      done(function (err) {
+        if (err) throw err
+
+        pull(sbot.patchwork.createInboxStream(), pull.collect(function (err, msgs) {
+          if (err) throw err
+          t.equal(msgs.length, 1)
+          t.equal(msgs[0].value.author, users.bob.id)
+          t.end()
+          sbot.close()
+        }))
+      })
+    })
+  })
+})
+
+tape('inbox index includes mentions of the user from followeds', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    var done = multicb()
+    users.bob.add({ type: 'post', text: 'hello from bob', mentions: [users.alice.id] }, done())
+    users.charlie.add({ type: 'post', text: 'hello from charlie', mentions: [users.alice.id] }, done())
+    done(function (err) {
+      if (err) throw err
+
+      pull(sbot.patchwork.createInboxStream(), pull.collect(function (err, msgs) {
+        if (err) throw err
+        t.equal(msgs.length, 1)
+        t.equal(msgs[0].value.author, users.bob.id)
+        t.end()
+        sbot.close()
+      }))
+    })
+  })
+})
+
+tape('inbox index counts correctly track read/unread', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    var done = multicb()
+    users.bob.add({ type: 'post', text: 'hello from bob', mentions: [users.alice.id] }, done())
+    users.charlie.add({ type: 'post', text: 'hello from charlie', mentions: [users.alice.id] }, done())
+    done(function (err, msgs) {
+      if (err) throw err
+      var inboxedMsg = msgs[0][1]
+
+      sbot.patchwork.getIndexCounts(function (err, counts) {
+        if (err) throw err
+        t.equal(counts.inbox, 1)
+        t.equal(counts.inboxUnread, 1)
+
+        sbot.patchwork.markRead(inboxedMsg.key, function (err) {
+          if (err) throw err
+
+          sbot.patchwork.getIndexCounts(function (err, counts) {
+            if (err) throw err
+            t.equal(counts.inbox, 1)
+            t.equal(counts.inboxUnread, 0)
+
+            sbot.patchwork.markUnread(inboxedMsg.key, function (err) {
+              if (err) throw err
+
+              sbot.patchwork.getIndexCounts(function (err, counts) {
+                if (err) throw err
+                t.equal(counts.inbox, 1)
+                t.equal(counts.inboxUnread, 1)
+
+                t.end()
+                sbot.close()
+              })
+            })
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('vote index includes upvotes on the users posts', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'post', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'vote', vote: { link: msg.key, value: 1 } }, done())
+      users.charlie.add({ type: 'vote', vote: { link: msg.key, value: 1 } }, done())
+      done(function (err) {
+        if (err) throw err
+
+        pull(sbot.patchwork.createVoteStream(), pull.collect(function (err, msgs) {
+          if (err) throw err
+          t.equal(msgs.length, 2)
+          t.end()
+          sbot.close()
+        }))
+      })
+    })
+  })
+})
+
+tape('vote index does not include downvotes, and removes unvotes', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'post', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'vote', vote: { link: msg.key, value: -1 } }, done())
+      users.charlie.add({ type: 'vote', vote: { link: msg.key, value: 1 } }, done())
+      users.charlie.add({ type: 'vote', vote: { link: msg.key, value: 0 } }, done())
+      done(function (err) {
+        if (err) throw err
+
+        pull(sbot.patchwork.createVoteStream(), pull.collect(function (err, msgs) {
+          if (err) throw err
+          t.equal(msgs.length, 0)
+          t.end()
+          sbot.close()
+        }))
+      })
+    })
+  })
+})
+
+tape('vote index counts correctly track read/unread', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'post', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'vote', vote: { link: msg.key, value: 1 } }, done())
+      users.charlie.add({ type: 'vote', vote: { link: msg.key, value: 1 } }, done())
+      done(function (err, msgs) {
+        if (err) throw err
+        var voteMsg = msgs[0][1]
+
+        sbot.patchwork.getIndexCounts(function (err, counts) {
+          if (err) throw err
+          t.equal(counts.votes, 2)
+          t.equal(counts.votesUnread, 2)
+
+          sbot.patchwork.markRead(voteMsg.key, function (err) {
+            if (err) throw err
+
+            sbot.patchwork.getIndexCounts(function (err, counts) {
+              if (err) throw err
+              t.equal(counts.votes, 2)
+              t.equal(counts.votesUnread, 1)
+
+              sbot.patchwork.markUnread(voteMsg.key, function (err) {
+                if (err) throw err
+
+                sbot.patchwork.getIndexCounts(function (err, counts) {
+                  if (err) throw err
+                  t.equal(counts.votes, 2)
+                  t.equal(counts.votesUnread, 2)
+                  t.end()
+                  sbot.close()
+                })
+              })
+            })
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('follow index includes all new followers', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: {},
+    bob: { follows: ['alice'] },
+    charlie: { follows: ['alice'] }
+  }, function (err, users) {
+    if (err) throw err
+
+    pull(sbot.patchwork.createFollowStream(), pull.collect(function (err, msgs) {
+      if (err) throw err
+      t.equal(msgs.length, 2)
+      t.end()
+      sbot.close()
+    }))
+  })
+})
+
+tape('follow index includes unfollows', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: {},
+    bob: { follows: ['alice'] },
+    charlie: { follows: ['alice'] }
+  }, function (err, users) {
+    if (err) throw err
+
+    users.charlie.add({ type: 'contact', contact: users.alice.id, following: false }, function (err) {
+      if (err) throw err
+      pull(sbot.patchwork.createFollowStream(), pull.collect(function (err, msgs) {
+        if (err) throw err
+        t.equal(msgs.length, 3)
+        t.end()
+        sbot.close()
+      }))
+    })
+  })
+})
+
+tape('follow index counts correctly track read/unread', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: {},
+    bob: { follows: ['alice'] },
+    charlie: { follows: ['alice'] }
+  }, function (err, users, msgs) {
+    if (err) throw err
+    var followMsg = msgs[1][1]
+
+    console.log('getting indexes 1')
+    sbot.patchwork.getIndexCounts(function (err, counts) {
+      if (err) throw err
+      t.equal(counts.follows, 2)
+      t.equal(counts.followsUnread, 2)
+
+      console.log('marking read')
+      sbot.patchwork.markRead(followMsg.key, function (err) {
+        if (err) throw err
+
+        console.log('getting indexes 2')
+        sbot.patchwork.getIndexCounts(function (err, counts) {
+          if (err) throw err
+          t.equal(counts.follows, 2)
+          t.equal(counts.followsUnread, 1)
+
+          sbot.patchwork.markUnread(followMsg.key, function (err) {
+            if (err) throw err
+
+            sbot.patchwork.getIndexCounts(function (err, counts) {
+              if (err) throw err
+              t.equal(counts.follows, 2)
+              t.equal(counts.followsUnread, 2)
+
+              t.end()
+              sbot.close()
+            })
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('home index includes all posts', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'post', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'post', text: 'hello from bob' }, done())
+      users.charlie.add({ type: 'post', text: 'hello from charlie', root: msg.key, branch: msg.key }, done())
+      done(function (err) {
+        if (err) throw err
+
+        pull(sbot.patchwork.createHomeStream(), pull.collect(function (err, msgs) {
+          if (err) throw err
+          t.equal(msgs.length, 3)
+          t.end()
+          sbot.close()
+        }))
+      })
+    })
+  })
+})
+
+tape('home index includes non-posts with post replies on them', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob'] }, // Note, does not follow charlie
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.alice.add({ type: 'nonpost', text: 'hello from alice' }, function (err, msg) {
+      if (err) throw err
+
+      var done = multicb()
+      users.bob.add({ type: 'nonpost', text: 'hello from bob' }, done())
+      users.charlie.add({ type: 'post', text: 'hello from charlie', root: msg.key, branch: msg.key }, done())
+      done(function (err) {
+        if (err) throw err
+
+        pull(sbot.patchwork.createHomeStream(), pull.collect(function (err, msgs) {
+          if (err) throw err
+          t.equal(msgs.length, 2)
+          t.equal(msgs[0].value.author, users.charlie.id)
+          t.equal(msgs[1].value.author, users.alice.id)
+          t.end()
+          sbot.close()
+        }))
+      })
+    })
+  })
+})
+

api/test/names.js

@@ -0,0 +1,220 @@
+var multicb = require('multicb')
+var tape    = require('tape')
+var u       = require('./util')
+
+tape('names default to self-assigned', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    sbot.patchwork.getNamesById(function (err, names) {
+      if (err) throw err
+      t.equal(names[users.alice.id], 'alice')
+      t.equal(names[users.bob.id], 'bob')
+      t.equal(names[users.charlie.id], 'charlie')
+
+      sbot.patchwork.getIdsByName(function (err, ids) {
+        if (err) throw err
+        t.equal(ids.alice, users.alice.id)
+        t.equal(ids.bob, users.bob.id)
+        t.equal(ids.charlie, users.charlie.id)
+        t.end()
+        sbot.close()
+      })
+    })
+  })
+})
+
+tape('the local users name assignments take precedence', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    var done = multicb()
+    users.alice.add({ type: 'about', about: users.bob.id, name: 'robert' }, done())
+    users.alice.add({ type: 'about', about: users.charlie.id, name: 'chuck' }, done())
+    done(function (err) {
+      if (err) throw err
+
+      sbot.patchwork.getNamesById(function (err, names) {
+        if (err) throw err
+        t.equal(names[users.alice.id], 'alice')
+        t.equal(names[users.bob.id], 'robert')
+        t.equal(names[users.charlie.id], 'chuck')
+
+        sbot.patchwork.getIdsByName(function (err, ids) {
+          if (err) throw err
+          t.equal(ids.alice, users.alice.id)
+          t.equal(ids.robert, users.bob.id)
+          t.equal(ids.chuck, users.charlie.id)
+          t.end()
+          sbot.close()
+        })
+      })
+    })
+  })
+})
+
+tape('conflicting names between followeds are tracked as action items', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.charlie.add({ type: 'about', about: users.charlie.id, name: 'bob' }, function (err) {
+      if (err) throw err
+
+      sbot.patchwork.getNamesById(function (err, names) {
+        if (err) throw err
+        t.equal(names[users.alice.id], 'alice')
+        t.equal(names[users.bob.id], 'bob')
+        t.equal(names[users.charlie.id], 'bob')
+
+        sbot.patchwork.getIdsByName(function (err, ids) {
+          if (err) throw err
+          t.equal(ids.alice, users.alice.id)
+          t.equal(ids.bob.length, 2)
+
+          sbot.patchwork.getActionItems(function (err, items) {
+            if (err) throw err
+            t.equal(items.bob.type, 'name-conflict')
+            t.equal(items.bob.name, 'bob')
+            t.equal(items.bob.ids.length, 2)
+            t.end()
+            sbot.close()
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('conflicting names are resolved by unfollowing', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.charlie.add({ type: 'about', about: users.charlie.id, name: 'bob' }, function (err) {
+      if (err) throw err
+
+      users.alice.add({ type: 'contact', contact: users.bob.id, following: false }, function (err) {
+        if (err) throw err
+
+        sbot.patchwork.getNamesById(function (err, names) {
+          if (err) throw err
+          t.equal(names[users.alice.id], 'alice')
+          t.equal(names[users.bob.id], 'bob')
+          t.equal(names[users.charlie.id], 'bob')
+
+          sbot.patchwork.getIdsByName(function (err, ids) {
+            if (err) throw err
+            t.equal(ids.alice, users.alice.id)
+            t.equal(ids.bob, users.charlie.id)
+
+            sbot.patchwork.getActionItems(function (err, items) {
+              if (err) throw err
+              t.equal(Object.keys(items).length, 0)
+              t.end()
+              sbot.close()
+            })
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('conflicting names are resolved by one of the users self-assigning a new name', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.charlie.add({ type: 'about', about: users.charlie.id, name: 'bob' }, function (err) {
+      if (err) throw err
+
+      users.bob.add({ type: 'about', about: users.bob.id, name: 'robert' }, function (err) {
+        if (err) throw err
+
+        sbot.patchwork.getNamesById(function (err, names) {
+          if (err) throw err
+          t.equal(names[users.alice.id], 'alice')
+          t.equal(names[users.bob.id], 'robert')
+          t.equal(names[users.charlie.id], 'bob')
+
+          sbot.patchwork.getIdsByName(function (err, ids) {
+            if (err) throw err
+            t.equal(ids.alice, users.alice.id)
+            t.equal(ids.robert, users.bob.id)
+            t.equal(ids.bob, users.charlie.id)
+
+            sbot.patchwork.getActionItems(function (err, items) {
+              if (err) throw err
+              t.equal(Object.keys(items).length, 0)
+              t.end()
+              sbot.close()
+            })
+          })
+        })
+      })
+    })
+  })
+})
+
+tape('conflicting names are resolved by the local user assigning a new name', function (t) {
+  var sbot = u.newserver()
+  u.makeusers(sbot, {
+    alice: { follows: ['bob', 'charlie'] },
+    bob: {},
+    charlie: {}
+  }, function (err, users) {
+    if (err) throw err
+
+    users.charlie.add({ type: 'about', about: users.charlie.id, name: 'bob' }, function (err) {
+      if (err) throw err
+
+      users.alice.add({ type: 'about', about: users.bob.id, name: 'robert' }, function (err) {
+        if (err) throw err
+
+        sbot.patchwork.getNamesById(function (err, names) {
+          if (err) throw err
+          t.equal(names[users.alice.id], 'alice')
+          t.equal(names[users.bob.id], 'robert')
+          t.equal(names[users.charlie.id], 'bob')
+
+          sbot.patchwork.getIdsByName(function (err, ids) {
+            if (err) throw err
+            t.equal(ids.alice, users.alice.id)
+            t.equal(ids.robert, users.bob.id)
+            t.equal(ids.bob, users.charlie.id)
+
+            sbot.patchwork.getActionItems(function (err, items) {
+              if (err) throw err
+              t.equal(Object.keys(items).length, 0)