Famedly release/v1.162

.github/workflows/ci.yaml

@@ -28,8 +28,8 @@ jobs:
           DOCKER_BUILDKIT: 1
         run: |
           mkdir -p homeserver
-          # Latest official dendrite release is still using Debian Stretch as a base, hence the specific commit
-          wget -O - "https://github.com/matrix-org/dendrite/archive/0489d16f95a3d9f1f5bc532e2060bd2482d7b156.tar.gz" | tar -xz --strip-components=1 -C homeserver
+          # Dendrite is unmaintained to just pin to a known working commit
+          wget -O - "https://github.com/element-hq/dendrite/archive/08cac1ccf0a45471132ad24a29e3ec15643675fa.tar.gz" | tar -xz --strip-components=1 -C homeserver
           (cd homeserver && docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile .)
           (cd cmd/homerunner/test && ./test.sh)
 

.github/workflows/zizmor.yaml

@@ -0,0 +1,24 @@
+# Taken from https://github.com/zizmorcore/zizmor-action/blob/f72bf176f67e8007f87b16d80f9880ece648aa65/README.md
+name: GitHub Actions Security Analysis with zizmor 🌈
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["**"]
+
+permissions: {}
+
+jobs:
+  zizmor:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Run zizmor 🌈
+        uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3

client/client.go

@@ -303,14 +303,14 @@ func (c *CSAPI) LeaveRoom(t ct.TestLike, roomID string) *http.Response {
 	return c.Do(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "leave"}, WithJSONBody(t, body))
 }
 
-// InviteRoom invites userID to the room ID, else fails the test.
+// MustInviteRoom invites userID to the room ID, else fails the test.
 func (c *CSAPI) MustInviteRoom(t ct.TestLike, roomID string, userID string) {
 	t.Helper()
 	res := c.InviteRoom(t, roomID, userID)
 	mustRespond2xx(t, res)
 }
 
-// InviteRoom invites userID to the room ID, else fails the test.
+// InviteRoom invites userID to the room ID.
 func (c *CSAPI) InviteRoom(t ct.TestLike, roomID string, userID string) *http.Response {
 	t.Helper()
 	// Invite the user to the room
@@ -320,6 +320,39 @@ func (c *CSAPI) InviteRoom(t ct.TestLike, roomID string, userID string) *http.Re
 	return c.Do(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "invite"}, WithJSONBody(t, body))
 }
 
+// MustKnockRoom will cause userID to knock on the room ID, else fails the test.
+//
+// Args:
+//   - `serverNames`: The list of servers to attempt to knock on the room through.
+//     These should be a resolvable address within the deployment network.
+func (c *CSAPI) MustKnockRoom(t ct.TestLike, roomID string, serverNames []spec.ServerName) {
+	t.Helper()
+	res := c.KnockRoom(t, roomID, serverNames)
+	mustRespond2xx(t, res)
+}
+
+// KnockRoom will cause userID to knock on the room ID.
+//
+// Args:
+//   - `serverNames`: The list of servers to attempt to knock on the room through.
+//     These should be a resolvable address within the deployment network.
+func (c *CSAPI) KnockRoom(t ct.TestLike, roomID string, serverNames []spec.ServerName) *http.Response {
+	t.Helper()
+	// construct URL query parameters
+	serverNameStrings := make([]string, len(serverNames))
+	for i, serverName := range serverNames {
+		serverNameStrings[i] = string(serverName)
+	}
+	query := url.Values{
+		"via": serverNameStrings,
+	}
+	// User knocks on the room
+	return c.Do(
+		t, "POST", []string{"_matrix", "client", "v3", "knock", roomID},
+		WithQueries(query), WithJSONBody(t, map[string]interface{}{}),
+	)
+}
+
 func (c *CSAPI) MustGetGlobalAccountData(t ct.TestLike, eventType string) *http.Response {
 	res := c.GetGlobalAccountData(t, eventType)
 	mustRespond2xx(t, res)

federation/server.go

@@ -18,6 +18,7 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"sync"
@@ -26,6 +27,7 @@ import (
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib/fclient"
 	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 
 	"github.com/gorilla/mux"
@@ -35,6 +37,9 @@ import (
 	"github.com/matrix-org/complement/config"
 	"github.com/matrix-org/complement/ct"
 	"github.com/matrix-org/complement/internal"
+	"github.com/matrix-org/complement/match"
+	"github.com/matrix-org/complement/must"
+	"github.com/matrix-org/complement/should"
 )
 
 // Subset of Deployment used in federation
@@ -64,8 +69,9 @@ type Server struct {
 
 	directoryHandlerSetup bool
 	aliases               map[string]string
-	rooms                 map[string]*ServerRoom
-	keyRing               *gomatrixserverlib.KeyRing
+	// List of rooms known to this server
+	rooms   map[string]*ServerRoom
+	keyRing *gomatrixserverlib.KeyRing
 }
 
 // EXPERIMENTAL
@@ -357,7 +363,7 @@ func (s *Server) MustCreateEvent(t ct.TestLike, room *ServerRoom, ev Event) goma
 	return pdu
 }
 
-// MustJoinRoom will make the server send a make_join and a send_join to join a room
+// MustJoinRoom will make the server send a /make_join and a /send_join to join a room
 // It returns the resultant room.
 //
 // Args:
@@ -443,6 +449,99 @@ func (s *Server) MustJoinRoom(t ct.TestLike, deployment FederationDeployment, re
 	return room
 }
 
+type knockRoom struct {
+	strictKnockRoomStateChecks bool
+}
+
+// KnockRoomOpt is an option for configuring how the server should knock on the room
+type KnockRoomOpt func(kr *knockRoom)
+
+// WithStrictKnockRoomStateChecks tells the server to strictly check that the received
+// `knock_room_state` is valid according to the spec (c.f. MSC4311).
+func WithStrictKnockRoomStateChecks() KnockRoomOpt {
+	return func(kr *knockRoom) {
+		kr.strictKnockRoomStateChecks = true
+	}
+}
+
+// MustKnockRoom  will make the server send a /make_knock and a /send_knock to knock on a room
+// It returns the resultant room.
+//
+// Args:
+//   - `remoteServer`: This should be a resolvable address within the deployment network.
+func (s *Server) MustKnockRoom(
+	t ct.TestLike,
+	deployment FederationDeployment,
+	remoteServer spec.ServerName,
+	roomID string,
+	userID string,
+	opts ...KnockRoomOpt,
+) *ServerRoom {
+	t.Helper()
+	var kr knockRoom
+	for _, opt := range opts {
+		opt(&kr)
+	}
+
+	origin := spec.ServerName(s.serverName)
+	fedClient := s.FederationClient(deployment)
+
+	makeKnockResp, err := fedClient.MakeKnock(context.Background(), origin, remoteServer, roomID, userID, SupportedRoomVersions())
+	if err != nil {
+		ct.Fatalf(t, "MustKnockRoom: make_knock failed: %v", err)
+	}
+
+	verImpl, err := gomatrixserverlib.GetRoomVersion(makeKnockResp.RoomVersion)
+	if err != nil {
+		ct.Fatalf(t, "MustKnockRoom: invalid room version: %v", err)
+	}
+
+	stateKey := userID
+	makeKnockResp.KnockEvent.SenderID = userID
+	makeKnockResp.KnockEvent.StateKey = &stateKey
+
+	eb := verImpl.NewEventBuilderFromProtoEvent(&makeKnockResp.KnockEvent)
+	knockEvent, err := eb.Build(time.Now(), origin, s.KeyID, s.Priv)
+	if err != nil {
+		ct.Fatalf(t, "MustKnockRoom: failed to sign event: %v", err)
+	}
+
+	// FIXME: Use `fedClient.SendKnock()` once it supports full PDU's vs stripped state
+	sendKnockPath := "/_matrix/federation/v1/send_knock/" + url.PathEscape(roomID) + "/" + url.PathEscape(knockEvent.EventID())
+	sendKnockReq := fclient.NewFederationRequest("PUT", origin, remoteServer, sendKnockPath)
+	if err := sendKnockReq.SetContent(knockEvent); err != nil {
+		ct.Fatalf(t, "MustKnockRoom: failed to set send_knock content: %v", err)
+	}
+	var rawResponse json.RawMessage
+	if err := s.SendFederationRequest(context.Background(), t, deployment, sendKnockReq, &rawResponse); err != nil {
+		ct.Fatalf(t, "MustKnockRoom: send_knock failed: %v", err)
+	}
+	knockResponse := gjson.ParseBytes(rawResponse)
+
+	// Strictly check that the received `knock_room_state` is valid according to the spec
+	// (c.f. MSC4311).
+	if kr.strictKnockRoomStateChecks {
+		must.MatchGJSON(t, knockResponse,
+			match.JSONArraySome("knock_room_state", func(event gjson.Result) error {
+				// MSC4311 also mandates that `m.room.create` event is required
+				return should.MatchGJSON(event, match.JSONKeyEqual("type", "m.room.create"))
+			}),
+			match.JSONArrayEach("knock_room_state", func(event gjson.Result) error {
+				// Each event should have extra fields `origin_server_ts` that indicate we're
+				// seeing a full PDU and not just a "stripped state event"
+				return should.MatchGJSON(event, match.JSONKeyPresent("origin_server_ts"))
+			}),
+		)
+	}
+
+	room := NewServerRoom(makeKnockResp.RoomVersion, roomID)
+	s.rooms[room.RoomID] = room
+
+	t.Logf("Server.MustKnockRoom knocked on room ID %s", room.RoomID)
+
+	return room
+}
+
 // Leaves a room. If this is rejecting an invite then a make_leave request is made first, before send_leave.
 //
 // Args:

go.mod

@@ -1,18 +1,18 @@
 module github.com/matrix-org/complement
 
-go 1.25.0
+go 1.26.0
 
 require (
 	github.com/gorilla/mux v1.8.1
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20260716140101-4fe595dc7f58
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
-	github.com/moby/moby/api v1.55.0
-	github.com/moby/moby/client v0.5.1
-	github.com/sirupsen/logrus v1.10.1
+	github.com/moby/moby/api v1.56.0
+	github.com/moby/moby/client v0.6.0
+	github.com/sirupsen/logrus v1.10.2
 	github.com/tidwall/gjson v1.19.0
 	github.com/tidwall/sjson v1.2.5
-	golang.org/x/crypto v0.55.0
+	golang.org/x/crypto v0.56.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	gonum.org/v1/plot v0.17.0
 )

go.sum

@@ -61,10 +61,10 @@ github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE=
 github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
-github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
-github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
-github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=
-github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=
+github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ=
+github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
+github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs=
+github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ=
 github.com/oleiade/lane/v2 v2.0.0 h1:XW/ex/Inr+bPkLd3O240xrFOhUkTd4Wy176+Gv0E3Qw=
 github.com/oleiade/lane/v2 v2.0.0/go.mod h1:i5FBPFAYSWCgLh58UkUGCChjcCzef/MI7PlQm2TKCeg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -73,10 +73,10 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/shoenig/test v1.11.0 h1:NoPa5GIoBwuqzIviCrnUJa+t5Xb4xi5Z+zODJnIDsEQ=
 github.com/shoenig/test v1.11.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI=
-github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q=
-github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk=
-github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI=
-github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw=
+github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
+github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU=
 github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc=
@@ -102,11 +102,13 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
 go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
-golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
+golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
+golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
@@ -149,8 +151,6 @@ gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
 gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
 gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=

internal/docker/deployer.go

@@ -17,7 +17,6 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"log"
 	"net/http"
@@ -741,11 +740,5 @@ func (t *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 		req.URL.Host = newURL.Host
 	}
 	req.URL.Scheme = "https"
-	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{
-			ServerName:         hsName,
-			InsecureSkipVerify: true,
-		},
-	}
-	return transport.RoundTrip(req)
+	return t.Deployment.transportFor(hsName).RoundTrip(req)
 }

internal/docker/deployment.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"sync"
@@ -28,6 +29,31 @@ type Deployment struct {
 	HS               map[string]*HomeserverDeployment
 	Config           *config.Complement
 	localpartCounter atomic.Int64
+	// HTTP transports used by RoundTripper, keyed by homeserver.
+	// If we don't re-use transports, tests which speak federation to the homeserver will leak file descriptors (fd)
+	// for a period of time (the IdleConnTimeout) which can then exceed the fd limit on some runtimes e.g. macOS has
+	// a conservative 256 fd limit by default. This manifests as obscure errors like "Invalid request signature"
+	// because the `/key/server` request performed by gomatrixserverlib fails.
+	transports sync.Map
+}
+
+// transportFor returns the (shared) HTTP transport used to talk to the given homeserver.
+func (d *Deployment) transportFor(hsName string) *http.Transport {
+	if t, ok := d.transports.Load(hsName); ok {
+		return t.(*http.Transport)
+	}
+	t, _ := d.transports.LoadOrStore(hsName, &http.Transport{
+		TLSClientConfig: &tls.Config{
+			ServerName:         hsName,
+			InsecureSkipVerify: true,
+		},
+		// Explicitly set the max idle conns per host to ensure we bound how many file descriptors we use per-server.
+		MaxIdleConnsPerHost: 2,
+		// Set a limit for how long connections can remain idle (and consume file descriptors) for.
+		// By default this is 0 meaning unlimited.
+		IdleConnTimeout: 30 * time.Second,
+	})
+	return t.(*http.Transport)
 }
 
 // HomeserverDeployment represents a running homeserver in a container.