Merge branch 'master' into qr_14173

* master: (206 commits)
  fix /video 404 page - closes lichess-org#14262
  Move classification flairs (fixes lichess-org#14264 ) (lichess-org#14265)
  add puzzle theme translation keys to the streak page - closes lichess-org#14273
  fix ui storage must remove the birthday key on expiration
  rename and factor ui storage birthday key
  New Crowdin updates (lichess-org#14271)
  hard coded ttl always
  remove unused function
  persistent log ws failures
  switch from xz to zstd for artifacts
  fix deploy after artifacts v4 update
  update to potentially much faster upload-artifact v4
  scala tweak, use form3.submit
  remove code duplication in ublog post ranking
  make it a bit clearer which fields are required when recomputing upost rank
  fix ublog post ranking when there is no rankAdjustDays
  don't confuse user with mod in mod log
  simplify using post.created.by
  tweak default broadcast pull periods
  recover from missing broadcast game using index.json metadata - closes lichess-org#14270
  ...

.github/workflows/assets.yml

@@ -47,10 +47,9 @@ jobs:
       - run: ./ui/build --no-install -p
       - run: cd ui && pnpm run test && cd -
       - run: mkdir assets && mv public assets/ && cp bin/download-lifat LICENSE COPYING.md README.md assets/ && git log -n 1 --pretty=oneline > assets/commit.txt
-      - run: cd assets && tar -cvpJf ../assets.tar.xz . && cd -
-        env:
-          XZ_OPT: '-0'
-      - uses: actions/upload-artifact@v3
+      - run: cd assets && tar --zstd -cvpf ../assets.tar.zst . && cd -
+      - uses: actions/upload-artifact@v4
         with:
           name: lila-assets
-          path: assets.tar.xz
+          path: assets.tar.zst
+          compression-level: 0 # already compressed

.github/workflows/flair.yml

@@ -0,0 +1,16 @@
+name: Validate Flair
+
+on:
+  push:
+    paths:
+      - 'public/flair/**'
+  pull_request:
+    paths:
+      - 'public/flair/**'
+
+jobs:
+  validate-flair:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: ./bin/validate-flair public/flair/img/

.github/workflows/server.yml

@@ -41,10 +41,11 @@ jobs:
       - run: TZ=UTC git log -1 --date=iso-strict-local --pretty='format:app.version.commit = "%H"%napp.version.date = "%ad"%napp.version.message = """%s"""%n' | tee conf/version.conf
       - run: ./lila -Depoll=true "test;stage"
       - run: cp LICENSE COPYING.md README.md target/universal/stage && git log -n 1 --pretty=oneline > target/universal/stage/commit.txt
-      - run: cd target/universal/stage && tar -cvpJf ../../../lila-3.0.tar.xz . && cd -
+      - run: cd target/universal/stage && tar --zstd -cvpf ../../../lila-3.0.tar.zst . && cd -
         env:
-          XZ_OPT: '-0'
-      - uses: actions/upload-artifact@v3
+          ZSTD_LEVEL: 1 # most files are already zipped
+      - uses: actions/upload-artifact@v4
         with:
           name: lila-server
-          path: lila-3.0.tar.xz
+          path: lila-3.0.tar.zst
+          compression-level: 0 # already compressed

app/controllers/Api.scala

@@ -351,7 +351,7 @@ final class Api(
   def mobileGames = Scoped(_.Web.Mobile) { _ ?=> _ ?=>
     val ids = get("ids").so(_.split(',').take(50).toList) map GameId.take
     ids.nonEmpty.so:
-      env.round.roundSocket.getMany(ids).flatMap(env.round.mobile.json).map(JsonOk)
+      env.round.roundSocket.getMany(ids).flatMap(env.round.mobile.online).map(JsonOk)
   }
 
   def ApiRequest(js: Context ?=> Fu[ApiResult]) = Anon:

app/controllers/Appeal.scala

@@ -31,7 +31,12 @@ final class Appeal(env: Env, reportC: => report.Report, prismicC: => Prismic, us
   )(using Context)(using me: Me): Fu[Frag] = env.appeal.api.byId(me) flatMap {
     case None =>
       renderAsync:
-        env.playban.api.currentBan(me).dmap(_.isDefined) map { html.appeal.tree(me, _) }
+        for
+          playban <- env.playban.api.currentBan(me).dmap(_.isDefined)
+          // if no blog, consider it's visible because even if it is not, for now the user
+          // has not been negatively impacted
+          ublogIsVisible <- env.ublog.api.isBlogVisible(me.userId).dmap(_.getOrElse(true))
+        yield html.appeal.tree(me, playban, ublogIsVisible)
     case Some(a) => renderPage(html.appeal.discussion(a, me, err | userForm))
   }
 

app/controllers/DailyFeed.scala

@@ -14,13 +14,10 @@ final class DailyFeed(env: Env) extends LilaController(env):
 
   def index = Open:
     for
-      updates <- api.recent(Max(50))
+      updates <- api.recent
       page    <- renderPage(html.dailyFeed.index(updates))
     yield Ok(page)
 
-  private def get(day: String): Fu[Option[Update]] =
-    scala.util.Try(LocalDate.parse(day)).toOption.so(api.get)
-
   def createForm = Secure(_.DailyFeed) { _ ?=> _ ?=>
     Ok.pageAsync(html.dailyFeed.create(api.form(none)))
   }
@@ -31,31 +28,34 @@ final class DailyFeed(env: Env) extends LilaController(env):
       .bindFromRequest()
       .fold(
         err => BadRequest.pageAsync(html.dailyFeed.create(err)),
-        up => api.set(up, none) inject Redirect(routes.DailyFeed.edit(up.day)).flashSuccess
+        data =>
+          val up = data toUpdate none
+          api.set(up) inject Redirect(routes.DailyFeed.edit(up.id)).flashSuccess
       )
   }
 
-  def edit(day: String) = Secure(_.DailyFeed) { _ ?=> _ ?=>
-    Found(get(day)): up =>
+  def edit(id: String) = Secure(_.DailyFeed) { _ ?=> _ ?=>
+    Found(api.get(id)): up =>
       Ok.pageAsync(html.dailyFeed.edit(api.form(up.some), up))
   }
 
-  def update(day: String) = SecureBody(_.DailyFeed) { _ ?=> _ ?=>
-    Found(get(day)): from =>
+  def update(id: String) = SecureBody(_.DailyFeed) { _ ?=> _ ?=>
+    Found(api.get(id)): from =>
       api
         .form(from.some)
         .bindFromRequest()
         .fold(
           err => BadRequest.pageAsync(html.dailyFeed.edit(err, from)),
-          up => api.set(up, from.some) inject Redirect(routes.DailyFeed.edit(up.day)).flashSuccess
+          data =>
+            api.set(data toUpdate from.id.some) inject Redirect(routes.DailyFeed.edit(from.id)).flashSuccess
         )
   }
 
-  def delete(day: String) = Secure(_.DailyFeed) { _ ?=> _ ?=>
-    Found(get(day)): up =>
-      api.delete(up.day) inject Redirect(routes.DailyFeed.index).flashSuccess
+  def delete(id: String) = Secure(_.DailyFeed) { _ ?=> _ ?=>
+    Found(api.get(id)): up =>
+      api.delete(up.id) inject Redirect(routes.DailyFeed.index).flashSuccess
   }
 
   def atom = Anon:
-    api.recent(Max(50)) map: ups =>
+    api.recentPublished map: ups =>
       Ok(html.dailyFeed.atom(ups)) as XML

app/controllers/Dev.scala

@@ -33,7 +33,10 @@ final class Dev(env: Env) extends LilaController(env):
     env.tutor.nbAnalysisSetting,
     env.tutor.parallelismSetting,
     env.firefoxOriginTrial,
-    env.credentiallessUaRegex
+    env.credentiallessUaRegex,
+    env.relay.proxyDomainRegex,
+    env.relay.proxyHostPort,
+    env.relay.proxyCredentials
   )
 
   def settings = Secure(_.Settings) { _ ?=> _ ?=>

app/controllers/Game.scala

@@ -28,10 +28,10 @@ final class Game(env: Env, apiC: => Api) extends LilaController(env):
       else Redirect(routes.Round.watcher(game.id, game.naturalOrientation.name))
   }
 
-  def exportOne(id: GameAnyId) = Anon:
+  def exportOne(id: GameAnyId) = AnonOrScoped():
     exportGame(id.gameId)
 
-  private[controllers] def exportGame(gameId: GameId)(using req: RequestHeader): Fu[Result] =
+  private[controllers] def exportGame(gameId: GameId)(using Context): Fu[Result] =
     env.round.proxyRepo.gameIfPresent(gameId) orElse env.game.gameRepo.game(gameId) flatMap {
       case None => NotFound
       case Some(game) =>
@@ -54,17 +54,17 @@ final class Game(env: Env, apiC: => Api) extends LilaController(env):
 
   private def handleExport(username: UserStr)(using ctx: Context) =
     env.user.repo byId username flatMap {
-      _.filter(u => u.enabled.yes || ctx.me.exists(_ is u) || isGrantedOpt(_.GamesModView)) so { user =>
+      _.filter(u => u.enabled.yes || ctx.is(u) || isGrantedOpt(_.GamesModView)) so { user =>
         val format = GameApiV2.Format byRequest req
         import lila.rating.{ Perf, PerfType }
         WithVs: vs =>
           env.security.ipTrust
-            .throttle(MaxPerSecond(ctx.me match
-              case Some(m) if m is lila.user.User.explorerId => env.apiExplorerGamesPerSecond.get()
-              case Some(m) if m is user.id                   => 60
-              case Some(_) if ctx.isOAuth => 30 // bonus for oauth logged in only (not for CSRF)
-              case _                      => 25
-            ))
+            .throttle(MaxPerSecond:
+              if ctx is lila.user.User.explorerId then env.apiExplorerGamesPerSecond.get()
+              else if ctx is user then 60
+              else if ctx.isOAuth then 30 // bonus for oauth logged in only (not for CSRF)
+              else 25
+            )
             .flatMap: perSecond =>
               val finished = getBoolOpt("finished") | true
               val config = GameApiV2.ByUserConfig(
@@ -87,7 +87,7 @@ final class Game(env: Env, apiC: => Api) extends LilaController(env):
                 ongoing = getBool("ongoing") || !finished,
                 finished = finished
               )
-              if ctx.me.exists(_ is lila.user.User.explorerId) then
+              if ctx.is(lila.user.User.explorerId) then
                 Ok.chunked(env.api.gameApiV2.exportByUser(config))
                   .pipe(noProxyBuffer)
                   .as(gameContentType(config))

app/controllers/GameMod.scala

@@ -77,7 +77,7 @@ final class GameMod(env: Env)(using akka.stream.Materializer) extends LilaContro
       }.parallel >> env.fishnet.awaiter(games.map(_.id), 2 minutes)
     } inject NoContent
 
-  private def downloadPgn(user: lila.user.User, gameIds: Seq[GameId]) =
+  private def downloadPgn(user: lila.user.User, gameIds: Seq[GameId])(using Option[Me]) =
     Ok.chunked {
       env.api.gameApiV2.exportByIds(
         GameApiV2.ByIdsConfig(

app/controllers/LilaController.scala

@@ -238,7 +238,7 @@ abstract private[controllers] class LilaController(val env: Env)
   def handleScopedFail(accepted: EndpointScopes, e: OAuthServer.AuthError)(using RequestHeader) = e match
     case e @ lila.oauth.OAuthServer.MissingScope(available) =>
       OAuthServer.responseHeaders(accepted, available):
-        Forbidden(jsonError(e.message))
+        forbiddenJson(e.message)
     case e =>
       OAuthServer.responseHeaders(accepted, TokenScopes(Nil)):
         Unauthorized(jsonError(e.message))
@@ -331,6 +331,8 @@ abstract private[controllers] class LilaController(val env: Env)
     .flatMap:
       f(using _)
 
+  given (using req: RequestHeader): lila.chat.AllMessages = lila.chat.AllMessages(HTTPRequest.isLitools(req))
+
   /* We roll our own action, as we don't want to compose play Actions. */
   private def action[A](parser: BodyParser[A])(handler: Request[A] ?=> Fu[Result]): EssentialAction = new:
     import play.api.libs.streams.Accumulator

app/controllers/Mod.scala

@@ -195,13 +195,16 @@ final class Mod(
     }
   }
 
-  def createNameCloseVote(username: UserStr) = SendToZulip(username, env.irc.api.nameCloseVote)
-  def askUsertableCheck(username: UserStr)   = SendToZulip(username, env.irc.api.usertableCheck)
+  def createNameCloseVote(username: UserStr) = Secure(_.SendToZulip) { _ ?=> me ?=>
+    env.report.api.inquiries ofModId me.id map {
+      _.filter(_.reason == lila.report.Reason.Username).map(_.bestAtom.simplifiedText)
+    } flatMap: reason =>
+      env.user.repo byId username orNotFound { env.irc.api.nameCloseVote(_, reason) inject NoContent }
 
-  private def SendToZulip(username: UserStr, method: UserModel => Me ?=> Funit) =
-    Secure(_.SendToZulip) { _ ?=> _ ?=>
-      env.user.repo byId username orNotFound { method(_) inject NoContent }
-    }
+  }
+  def askUsertableCheck(username: UserStr) = Secure(_.SendToZulip) { _ ?=> _ ?=>
+    env.user.repo byId username orNotFound { env.irc.api.usertableCheck(_) inject NoContent }
+  }
 
   def table = Secure(_.Admin) { ctx ?=> _ ?=>
     Ok.pageAsync:

app/controllers/Push.scala

@@ -5,11 +5,11 @@ import lila.push.WebSubscription
 
 final class Push(env: Env) extends LilaController(env):
 
-  def mobileRegister(platform: String, deviceId: String) = Auth { ctx ?=> me ?=>
+  def mobileRegister(platform: String, deviceId: String) = AuthOrScoped(_.Web.Mobile) { ctx ?=> me ?=>
     env.push.registerDevice(me, platform, deviceId) inject NoContent
   }
 
-  def mobileUnregister = Auth { ctx ?=> me ?=>
+  def mobileUnregister = AuthOrScoped(_.Web.Mobile) { ctx ?=> me ?=>
     env.push.unregisterDevices(me) inject NoContent
   }
 

app/controllers/RelayRound.scala

@@ -9,6 +9,8 @@ import lila.common.HTTPRequest
 import lila.relay.{ RelayRound as RoundModel, RelayRoundForm, RelayTour as TourModel }
 import chess.format.pgn.PgnStr
 import views.*
+import lila.common.config.{ Max, MaxPerSecond }
+import play.api.libs.json.Json
 
 final class RelayRound(
     env: Env,
@@ -42,12 +44,11 @@ final class RelayRound(
               ),
             setup =>
               rateLimitCreation(whenRateLimited):
-                env.relay.api.create(setup, tour) flatMap { round =>
+                env.relay.api.create(setup, tour) flatMap: rt =>
                   negotiate(
-                    Redirect(routes.RelayRound.show(tour.slug, round.slug, round.id.value)),
-                    JsonOk(env.relay.jsonView.withUrl(round withTour tour))
+                    Redirect(routes.RelayRound.show(tour.slug, rt.relay.slug, rt.relay.id)),
+                    JsonOk(env.relay.jsonView.myRound(rt))
                   )
-                }
           )
   }
 
@@ -101,18 +102,28 @@ final class RelayRound(
             else env.study.api byIdWithChapter rt.round.studyId
           sc orNotFound { doShow(rt, _) }
         ,
-        json = Found(env.relay.api.byIdWithTour(id)): rt =>
-          Found(env.study.studyRepo.byId(rt.round.studyId)): study =>
-            studyC.CanView(study)(
-              env.study.chapterRepo orderedMetadataByStudy rt.round.studyId map { games =>
-                JsonOk(env.relay.jsonView.withUrlAndGames(rt, games))
-              }
-            )(studyC.privateUnauthorizedJson, studyC.privateForbiddenJson)
+        json = doApiShow(id)
       )
 
+  def apiShow(ts: String, rs: String, id: RelayRoundId) = AnonOrScoped(_.Study.Read):
+    doApiShow(id)
+
+  private def doApiShow(id: RelayRoundId)(using Context): Fu[Result] =
+    Found(env.relay.api.byIdWithTour(id)): rt =>
+      Found(env.study.studyRepo.byId(rt.round.studyId)): study =>
+        studyC.CanView(study)(
+          env.study.chapterRepo orderedMetadataByStudy rt.round.studyId map: games =>
+            JsonOk(env.relay.jsonView.withUrlAndGames(rt withStudy study, games))
+        )(studyC.privateUnauthorizedJson, studyC.privateForbiddenJson)
+
   def pgn(ts: String, rs: String, id: StudyId) = studyC.pgn(id)
   def apiPgn                                   = studyC.apiPgn
 
+  def apiMyRounds = Scoped(_.Study.Read) { ctx ?=> _ ?=>
+    val source = env.relay.api.myRounds(MaxPerSecond(20), getIntAs[Max]("nb")).map(env.relay.jsonView.myRound)
+    apiC.GlobalConcurrencyLimitPerIP.download(ctx.ip)(source)(apiC.sourceToNdJson)
+  }
+
   def stream(id: RelayRoundId) = AnonOrScoped(): ctx ?=>
     Found(env.relay.api.byIdWithStudy(id)): rt =>
       studyC.CanView(rt.study) {
@@ -127,10 +138,16 @@ final class RelayRound(
 
   def push(id: RelayRoundId) = ScopedBody(parse.tolerantText)(Seq(_.Study.Write)) { ctx ?=> me ?=>
     env.relay.api
-      .byIdAndContributor(id)
+      .byIdWithStudy(id)
       .flatMap:
-        case None     => notFoundJson()
-        case Some(rt) => env.relay.push(rt, PgnStr(ctx.body.body)) inject jsonOkResult
+        case None                                    => notFoundJson()
+        case Some(rt) if !rt.study.canContribute(me) => forbiddenJson()
+        case Some(rt) =>
+          env.relay
+            .push(rt.withTour, PgnStr(ctx.body.body))
+            .map:
+              case Right(moves) => JsonOk(Json.obj("moves" -> moves))
+              case Left(e)      => JsonBadRequest(e.message)
   }
 
   private def WithRoundAndTour(@nowarn ts: String, @nowarn rs: String, id: RelayRoundId)(

app/controllers/Setup.scala

@@ -78,7 +78,7 @@ final class Setup(
                     val message = lila.challenge.ChallengeDenied.translated(denied)
                     negotiate(
                       // 403 tells setupCtrl.ts to close the setup modal
-                      Forbidden(jsonError(message)), // TODO test
+                      forbiddenJson(message), // TODO test
                       BadRequest(jsonError(message))
                     )
                   case None =>

app/controllers/Streamer.scala

@@ -24,10 +24,10 @@ final class Streamer(env: Env, apiC: => Api) extends LilaController(env):
           page        <- renderPage(html.streamer.index(live, pager, requests))
         yield Ok(page)
 
-  def featured = Anon:
+  def featured = Anon: ctx ?=>
     env.streamer.liveStreamApi.all.map: streams =>
       val max      = env.streamer.homepageMaxSetting.get()
-      val featured = streams.homepage(max, req, none) withTitles env.user.lightUserApi
+      val featured = streams.homepage(max, ctx.acceptLanguages) withTitles env.user.lightUserApi
       JsonOk:
         featured.live.streams.map: s =>
           Json.obj(

app/controllers/Study.scala

@@ -539,7 +539,7 @@ final class Study(
   )
 
   def privateForbiddenText = Forbidden("This study is now private")
-  def privateForbiddenJson = Forbidden(jsonError("This study is now private"))
+  def privateForbiddenJson = forbiddenJson("This study is now private")
   def privateForbiddenFu(study: StudyModel)(using Context) = negotiate(
     Forbidden.page(html.site.message.privateStudy(study)),
     privateForbiddenJson