Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt

docs/api/swagger.yaml

@@ -1562,6 +1562,64 @@ definitions:
     type: string
     x-go-name: Visibility
     x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model
+  swaggerStatusRepliesCollection:
+    properties:
+      '@context':
+        description: ActivityStreams context.
+        example: https://www.w3.org/ns/activitystreams
+        type: string
+        x-go-name: Context
+      first:
+        $ref: '#/definitions/swaggerStatusRepliesCollectionPage'
+      id:
+        description: ActivityStreams ID.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies
+        type: string
+        x-go-name: ID
+      type:
+        description: ActivityStreams type.
+        example: Collection
+        type: string
+        x-go-name: Type
+    title: SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies.
+    type: object
+    x-go-name: SwaggerStatusRepliesCollection
+    x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user
+  swaggerStatusRepliesCollectionPage:
+    properties:
+      id:
+        description: ActivityStreams ID.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true
+        type: string
+        x-go-name: ID
+      items:
+        description: Items on this page.
+        example:
+        - https://example.org/users/some_other_user/statuses/086417595981111564
+        - https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R
+        items:
+          type: string
+        type: array
+        x-go-name: Items
+      next:
+        description: Link to the next page.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true
+        type: string
+        x-go-name: Next
+      partOf:
+        description: Collection this page belongs to.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies
+        type: string
+        x-go-name: PartOf
+      type:
+        description: ActivityStreams type.
+        example: CollectionPage
+        type: string
+        x-go-name: Type
+    title: SwaggerStatusRepliesCollectionPage represents one page of a collection.
+    type: object
+    x-go-name: SwaggerStatusRepliesCollectionPage
+    x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user
   tag:
     properties:
       name:
@@ -1621,7 +1679,7 @@ info:
     name: AGPL3
     url: https://www.gnu.org/licenses/agpl-3.0.en.html
   title: GoToSocial
-  version: 0.1.0-SNAPSHOT
+  version: 0.1.0-SNAPSHOT-dereference_remote_replies
 paths:
   /api/v1/accounts:
     post:
@@ -2395,11 +2453,10 @@ paths:
       - blocks
   /api/v1/instance:
     get:
-      description: "This is mostly provided for Mastodon application compatibility,
-        since many apps that work with Mastodon use `/api/v1/instance` to inform their
-        connection parameters. \n\nHowever, it can also be used by other instances
-        for gathering instance information and representing instances in some UI or
-        other."
+      description: |-
+        This is mostly provided for Mastodon application compatibility, since many apps that work with Mastodon use `/api/v1/instance` to inform their connection parameters.
+
+        However, it can also be used by other instances for gathering instance information and representing instances in some UI or other.
       operationId: instanceGet
       produces:
       - application/json
@@ -3306,6 +3363,56 @@ paths:
       summary: See public statuses/posts that your instance is aware of.
       tags:
       - timelines
+  /users/{username}/statuses/{status}/replies:
+    get:
+      description: |-
+        Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`.
+
+        If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`.
+
+        HTTP signature is required on the request.
+      operationId: s2sRepliesGet
+      parameters:
+      - description: Username of the account.
+        in: path
+        name: username
+        required: true
+        type: string
+      - description: ID of the status.
+        in: path
+        name: status
+        required: true
+        type: string
+      - default: false
+        description: Return response as a CollectionPage.
+        in: query
+        name: page
+        type: boolean
+      - default: false
+        description: Return replies only from accounts other than the status owner.
+        in: query
+        name: only_other_accounts
+        type: boolean
+      - description: Minimum ID of the next status, used for paging.
+        in: query
+        name: min_id
+        type: string
+      produces:
+      - application/activity+json
+      responses:
+        "200":
+          description: ""
+          schema:
+            $ref: '#/definitions/swaggerStatusRepliesCollection'
+        "401":
+          description: unauthorized
+        "403":
+          description: forbidden
+        "404":
+          description: not found
+      summary: Get the replies collection for a status.
+      tags:
+      - s2s/federation
 schemes:
 - https
 - http

docs/assets/diagrams/conversation_thread.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2021-08-09T09:38:01.312Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.0.3 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="jpDbRW_Z_HbkNN_9kqIg" version="13.0.3" type="device"><diagram id="xfbHBLe4vMsijS9wwlE7" name="Page-1">7VrBcpswEP0ajs0AMjY+xkncXtqmTaZNTxnFKEYTQIwQsZ2vrwQSBok0rmObmGTG40ELEtq3T29Xsi1wFi8/U5iGX0mAIsu1g6UFzi3XHfku/xaGlTI4pWFOcVCaaoYr/ISk0ZbWHAcoazzICIkYTpvGGUkSNGMNG6SULJqP3ZOo+dYUzpFhuJrByLT+xgELS6vv2Wv7F4TnoXqzY8s7MVQPS0MWwoAsaiZwYYEzSggrr+LlGYoEdgqXst/0mbvVxChK2CYdTp/8m+9T+/pXPP0Bw2sCycPtp0E5yiOMcumwNbAzEiOSIG6/DnEmhhBfUDyH6Er4hmCCk/tcDJ+SjJ1ID9lKwUZJngRIvNmxwGQRYoauUjgTdxecJ9wWsjiStzNGyUMFLwdmImeFKEPLZ911KhA5+RCfMysmJzsAhbsknuv5ZXuxDqOjnglrIRxKG5TMmVdDr8HlFxLf/8B6aGINpo3P61BU/LJ5I4BZWHXcAZq+BqYCqTMwRyaYOnooCU6FAvBWIuisoVKDDi0xu6ld/xEonniydb6UoBaNlWok3I2beqPWSzTX3YqW6ldOEwWG7Gih4K6QnM7Qy4RikM4Re2mRm6Gtxc5rCZ2yURRBhh+b022Lp3zDJcHckYo5LmhSZwA0SpRuyl51/dIH0jgIfG2gEgdjoIJeldvbM843GHdJ0SMmeSZVsG0F8xXHmlyDEZ4n/HrGw40oN4h1iXnCOZU3YhwEovuEogw/wbtiKMGcVHhW+OpNLO9cjJUzkpUpc0frvMq7aqGPzYU+biGLu691Pj5i0fSGby0Fqei+a9kcbCib4y5lczDSZNPZUjY9rQ4COrn2LJuOY1DumxDF/mimHqo2zRwdUjMdt0ei6fhdl5oO6BGcYNA5nC17zlfkoC3zyTa5q4McVC7lzpKQpyUhnRQbJyH7hWy27yTkfdQ9m3MOvKnCZ7Crwmd8YM6Z5z0/URqtelHy6OdBrplV2jiyv5LHPA/qLdjA6xpsc1M+IbyY51opq3rXvlsVUhAThm5NrT22EAAtf1WHWvUqanjIGKiBj7Io1c/mnTY4D1qUuuYu9XjhBKPO4WzZgb67emvT4/nar4FdnM9rJ75b1/iuttM8dI3vtmzTPzj3T73rjHP6dnBrzukb1ENzzjzL6E3Zqf9wB1p+8z1o2emam/jegl2p8u7B5s31n07KhbD+5w64+As=</diagram></mxfile>

docs/federation/behaviors/conversation_threads.md

@@ -0,0 +1,45 @@
+# Conversation Threads
+
+Due to the nature of decentralization and federation, it is practically impossible for any one server on the fediverse to be aware of every post in a given conversation thread.
+
+With that said, it is possible to do 'best effort' dereferencing of threads, whereby remote replies are fetched from one server onto another, to try to more fully flesh out a conversation.
+
+GoToSocial does this by iterating up and down the thread of a conversation, pulling in remote statuses where possible.
+
+## Example
+
+Let's say we have two accounts: `local_account` on `our.server`, and `remote_1` on `remote.1`.
+
+In this scenario, `local_account` follows `remote_1`, so posts from `remote_1` show up in the home timeline of `local_account`.
+
+Now, `remote_1` boosts/reblogs a post from a third account, `remote_2`, residing on server `remote.2`.
+
+`local_account` does not follow `remote_2`, and neither does anybody else on `our.server`, which means that `our.server` has not seen this post by `remote_2` before.
+
+![A diagram of the conversation thread, showing the post from remote_2, and possible ancestor and descendant posts](../../assets/diagrams/conversation_thread.png)
+
+What GoToSocial will do now, is 'dereference' the post by `remote_2` to check if it is part of a thread and, if so, whether any other parts of the thread can be obtained.
+
+GtS begins by checking the `inReplyTo` property of the post, which is set when a post is a reply to another post. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-inreplyto). If `inReplyTo` is set, GoToSocial derefences the replied-to post. If *this* post also has an `inReplyTo` set, then GoToSocial dereferences that too, and so on.
+
+Once all of these **ancestors** of a status have been retrieved, GtS will begin working down through the **descendants** of posts.
+
+It does this by checking the `replies` property of a derefenced post, and working through replies, and replies of replies. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-replies).
+
+This process of thread dereferencing will likely involve making multiple HTTP calls to different servers, especially if the thread is long and complicated.
+
+The end result of this dereferencing is that, assuming the reblogged post by `remote_2` was part of a thread, then `local_account` should now be able to see posts in the thread when they open the status on their home timeline. In other words, they will see replies from accounts on other servers (who they may not have come across yet), in addition to any previous and next posts in the thread as posted by `remote_2`.
+
+This gives `local_account` a more complete view on the conversation, as opposed to just seeing the reblogged post in isolation and out of context. It also gives `local_account` the opportunity to discover new accounts to follow, based on replies to `remote_2`.
+
+## Privacy and Security
+
+During the dereferencing process, GoToSocial signs outgoing requests using the key of the actor who received the activity that necessitated dereferencing. To use the above example, this means that all dereferencing requests would be signed by `local_account`. This gives remote servers the ability to refuse these dereferencing requests, assuming that `local_account` is blocked by one or more participants in the conversation.
+
+From GoToSocial's side, domain blocks will be respected during the dereferencing process, to avoid making calls to servers that `our.server` has blocked.
+
+Individual account blocks will also be respected, meaning that `our.server` won't try to dereference posts from accounts blocked by `local_account`.
+
+Finally, GoToSocial expects that remote servers will only list replies that are marked as public (either `to` or `cc`). GtS may *try* to dereference followers-only posts, but it will assume that remote servers will check whether or not `local_account` is allowed to view them, and refuse accordingly.
+
+Of course, when `local_account` opens up the conversation thread in whatever application they are using, GoToSocial will apply the usual post visibility filtering to ensure that they do not see any posts that they shouldn't have access to.

docs/federation/glossary.md

@@ -0,0 +1,27 @@
+# Glossary
+
+Some commonly-used terms in discussions of federation, and their meanings.
+
+### `ActivityPub`
+
+A decentralized social networking protocol based on the ActivityStreams data format. See [here](https://www.w3.org/TR/activitypub/).
+
+GoToSocial uses the ActivityPub protocol to communicate between GtS servers, and with other federated servers like Mastodon, Pixelfed, etc.
+
+### `ActivityStreams`
+
+A model/data format for representing potential and completed activities using JSON. See [here](https://www.w3.org/TR/activitystreams-core/).
+
+GoToSocial uses the ActivityStreams data model to 'speak' ActivityPub with other servers.
+
+### `Actor`
+
+An actor is an ActivityStreams object that is capable of performing some Activity like following, liking, creating a post, reblogging, etc. See [here](https://www.w3.org/TR/activitypub/#actors).
+
+In GoToSocial, each account/user is an actor.
+
+### `Dereference`
+
+To 'dereference' a post or a profile means to make an HTTP call to the server that hosts that post or profile, in order to obtain its ActivityStreams representation.
+
+GoToSocial 'dereferences' posts and profiles on remote servers, in order to convert them to models that GoToSocial can understand and work with.

docs/federation/principles.md

@@ -0,0 +1,9 @@
+# Principles
+
+TODO -- describe the principles GtS uses for federating.
+
+Eg:
+
+* Why federate?
+* Why ActivityPub?
+* Broad overview of how GtS fits into the fediverse.

docs/federation/security.md

@@ -0,0 +1,7 @@
+# Security
+
+TODO: describe the security model we use for federation.
+
+* http signatures
+* behavior for refusing requests
+* how data is protected