Merge pull request #4273 from thematters/develop

Release: v5.9.0

.env.example

@@ -29,7 +29,7 @@ MATTERS_PG_HOST=db
 MATTERS_PG_USER=postgres
 MATTERS_PG_PASSWORD=postgres
 MATTERS_PG_DATABASE=matters-dev
-MATTERS_PG_READONLY_CONNECTION_STRING=postgres://postgres@db:postgres/matters-dev
+MATTERS_PG_READONLY_CONNECTION_STRING=postgresql://postgres:postgres@db:5432/matters-dev
 MATTERS_TSQI_SERVER_URL=http://localhost:2883
 
 MATTERS_SEARCH_PG_CONNECTION_STRING=postgres://postgres@db/progress
@@ -96,3 +96,4 @@ MATTERS_PASSPHRASES_API_URL=
 MATTERS_PASSPHRASES_SECRET=
 
 MATTERS_SPAM_DETECTION_API_URL=
+MATTERS_CHANNEL_CLASSIFICATION_API_URL=

db/migrations/20241226155054_channel.js

@@ -0,0 +1,75 @@
+const { baseDown } = require('../utils')
+
+const channel_table = 'channel'
+const article_channel_table = 'article_channel'
+const article_channel_job_table = 'article_channel_job'
+const feature_flag_table = 'feature_flag'
+
+exports.up = async (knex) => {
+  // Create channel table
+  await knex('entity_type').insert({ table: channel_table })
+  await knex.schema.createTable(channel_table, (t) => {
+    t.bigIncrements('id').primary()
+    t.string('name').notNullable()
+    t.string('description')
+    t.string('provider_id').notNullable().unique()
+    t.boolean('enabled').defaultTo(true).notNullable()
+    t.timestamp('created_at').defaultTo(knex.fn.now()).notNullable()
+    t.timestamp('updated_at').defaultTo(knex.fn.now()).notNullable()
+
+    t.index('provider_id').index('enabled')
+  })
+
+  // Create article_channel junction table
+  await knex('entity_type').insert({ table: article_channel_table })
+  await knex.schema.createTable(article_channel_table, (t) => {
+    t.bigIncrements('id').primary()
+    t.bigInteger('article_id').unsigned().notNullable()
+    t.bigInteger('channel_id').unsigned().notNullable()
+
+    t.float('score').nullable()
+
+    // admin fields
+    t.boolean('enabled').defaultTo(true).notNullable()
+    t.boolean('is_labeled').defaultTo(false).notNullable()
+
+    t.timestamp('created_at').defaultTo(knex.fn.now()).notNullable()
+    t.timestamp('updated_at').defaultTo(knex.fn.now()).notNullable()
+
+    t.foreign('article_id').references('id').inTable('article')
+    t.foreign('channel_id').references('id').inTable('channel')
+
+    t.unique(['article_id', 'channel_id'])
+    t.index('article_id')
+      .index('channel_id')
+      .index('score')
+      .index('is_labeled')
+      .index('enabled')
+  })
+
+  // Create article_channel_job junction table
+  await knex('entity_type').insert({ table: article_channel_job_table })
+  await knex.schema.createTable(article_channel_job_table, (t) => {
+    t.bigIncrements('id').primary()
+    t.bigInteger('article_id').unsigned().notNullable().unique()
+    t.string('job_id').notNullable().unique()
+    t.enum('state', ['processing', 'finished', 'error']).notNullable()
+
+    t.timestamp('created_at').defaultTo(knex.fn.now()).notNullable()
+    t.timestamp('updated_at').defaultTo(knex.fn.now()).notNullable()
+
+    t.foreign('article_id').references('id').inTable('article')
+  })
+
+  // Add feature flag
+  await knex(feature_flag_table).insert({
+    name: 'article_channel',
+    flag: 'off',
+    value: 0.5,
+  })
+}
+
+exports.down = async (knex) => {
+  await baseDown(article_channel_table)(knex)
+  await baseDown(channel_table)(knex)
+}

db/migrations/20250111212414_add_user_restriction_indexes.js

@@ -0,0 +1,15 @@
+const table = 'user_restriction'
+
+exports.up = async (knex) => {
+  await knex.schema.table(table, (t) => {
+    t.index('type')
+    t.unique(['user_id', 'type'])
+  })
+}
+
+exports.down = async (knex) => {
+  await knex.schema.table(table, (t) => {
+    t.dropUnique(['user_id', 'type'])
+    t.dropIndex('type')
+  })
+}

db/migrations/20250113101920_user_feature_flag.js

@@ -0,0 +1,19 @@
+const { baseDown } = require('../utils')
+
+const table = 'user_feature_flag'
+
+exports.up = async (knex) => {
+  await knex('entity_type').insert({ table })
+
+  await knex.schema.createTable(table, (t) => {
+    t.bigIncrements('id').primary()
+    t.bigInteger('user_id').unsigned().notNullable()
+    t.enu('type', ['bypassSpamDetection']).notNullable()
+    t.timestamp('created_at').defaultTo(knex.fn.now())
+
+    t.index(['user_id', 'type'])
+    t.foreign('user_id').references('id').inTable('user')
+  })
+}
+
+exports.down = baseDown(table)

db/migrations/20250120091037_add_retried_at_article_channel_job.js

@@ -0,0 +1,15 @@
+const { baseDown } = require('../utils')
+
+const article_channel_job_table = 'article_channel_job'
+
+exports.up = async (knex) => {
+  await knex.schema.table(article_channel_job_table, function (t) {
+    t.timestamp('retried_at')
+  })
+}
+
+exports.down = async (knex) => {
+  await knex.schema.table(article_channel_job_table, function (t) {
+    t.dropColumn('retried_at')
+  })
+}

db/seeds/31_feature_flag.js

@@ -36,5 +36,10 @@ exports.seed = async (knex) => {
       flag: 'off',
       value: 0.5,
     },
+    {
+      name: 'article_channel',
+      flag: 'off',
+      value: 0.5,
+    },
   ])
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matters-server",
-  "version": "5.8.0",
+  "version": "5.9.0",
   "description": "Matters Server",
   "author": "Matters <[email protected]>",
   "main": "build/index.js",

schema.graphql

@@ -34,6 +34,7 @@ type Query {
   exchangeRates(input: ExchangeRatesInput): [ExchangeRate!]
   oauthClient(input: OAuthClientInput!): OAuthClient
   moment(input: MomentInput!): Moment
+  channels: [Channel!]!
 }
 
 type Mutation {
@@ -142,6 +143,7 @@ type Mutation {
   putAnnouncement(input: PutAnnouncementInput!): Announcement!
   deleteAnnouncements(input: DeleteAnnouncementsInput!): Boolean!
   putRestrictedUsers(input: PutRestrictedUsersInput!): [User!]!
+  putUserFeatureFlags(input: PutUserFeatureFlagsInput!): [User!]!
   putIcymiTopic(input: PutIcymiTopicInput!): IcymiTopic
   setSpamStatus(input: SetSpamStatusInput!): Article!
 
@@ -274,6 +276,9 @@ type Mutation {
   deleteMoment(input: DeleteMomentInput!): Moment!
   likeMoment(input: LikeMomentInput!): Moment!
   unlikeMoment(input: UnlikeMomentInput!): Moment!
+  putChannel(input: PutChannelInput!): Channel!
+  setArticleChannels(input: SetArticleChannelsInput!): Article!
+  classifyArticlesChannels(input: ClassifyArticlesChannelsInput!): Boolean!
 }
 
 """
@@ -284,9 +289,6 @@ type Article implements Node & PinnableWork {
   """Unique ID of this article"""
   id: ID!
 
-  """The number represents how popular is this article."""
-  topicScore: Int
-
   """Slugified article title."""
   slug: String!
 
@@ -351,7 +353,6 @@ type Article implements Node & PinnableWork {
 
   """Related articles to this article."""
   relatedArticles(input: ConnectionArgs!): ArticleConnection!
-  relatedArticlesExcludeSpam(input: ConnectionArgs!): ArticleConnection!
 
   """Donation-related articles to this article."""
   relatedDonationArticles(input: RelatedDonationArticlesInput!): ArticleConnection!
@@ -442,6 +443,9 @@ type Article implements Node & PinnableWork {
 
   """associated campaigns"""
   campaigns: [ArticleCampaign!]!
+
+  """whether this article is noindex"""
+  noindex: Boolean!
   oss: ArticleOSS!
   remark: String
 
@@ -513,7 +517,6 @@ type Tag implements Node {
 
   """List of how many articles were attached with this tag."""
   articles(input: TagArticlesInput!): ArticleConnection!
-  articlesExcludeSpam(input: TagArticlesInput!): ArticleConnection!
 
   """Time of this tag was created."""
   createdAt: DateTime!
@@ -557,6 +560,7 @@ type ArticleOSS {
   inRecommendNewest: Boolean!
   inSearch: Boolean!
   spamStatus: SpamStatus!
+  channels: [ArticleChannel!]!
 }
 
 type SpamStatus {
@@ -569,6 +573,19 @@ type SpamStatus {
   isSpam: Boolean
 }
 
+type ArticleChannel {
+  channel: Channel!
+
+  """confident score by machine"""
+  score: Float
+
+  """whether this article is labeled by human, null for not labeled yet. """
+  isLabeled: Boolean!
+
+  """whether this article channel is enabled"""
+  enabled: Boolean!
+}
+
 type ArticleTranslation {
   title: String
   content: String
@@ -2116,6 +2133,11 @@ type UserRestriction {
   createdAt: DateTime!
 }
 
+type UserFeatureFlag {
+  type: UserFeatureFlagType!
+  createdAt: DateTime!
+}
+
 type Report implements Node {
   id: ID!
   reporter: User!
@@ -2282,6 +2304,11 @@ input PutRestrictedUsersInput {
   restrictions: [UserRestrictionType!]!
 }
 
+input PutUserFeatureFlagsInput {
+  ids: [ID!]!
+  flags: [UserFeatureFlagType!]!
+}
+
 input SubmitReportInput {
   targetId: ID!
   reason: ReportReason!
@@ -2379,6 +2406,7 @@ enum FeatureName {
   circle_management
   circle_interact
   spam_detection
+  article_channel
 }
 
 enum FeatureFlag {
@@ -2403,6 +2431,10 @@ enum UserRestrictionType {
   articleNewest
 }
 
+enum UserFeatureFlagType {
+  bypassSpamDetection
+}
+
 enum ReportReason {
   tort
   illegal_advertising
@@ -2474,35 +2506,30 @@ type Recommendation {
 
   """Global articles sort by publish time."""
   newest(input: ConnectionArgs!): ArticleConnection!
-  newestExcludeSpam(input: ConnectionArgs!): ArticleConnection!
 
   """Global articles sort by latest activity time."""
   hottest(input: ConnectionArgs!): ArticleConnection!
-  hottestExcludeSpam(input: ConnectionArgs!): ArticleConnection!
 
   """'In case you missed it' recommendation."""
   icymi(input: ConnectionArgs!): ArticleConnection!
 
   """'In case you missed it' topic."""
   icymiTopic: IcymiTopic
 
+  """Articles from a specific channel"""
+  channelArticles(input: ChannelArticlesInput!): ArticleConnection!
+
   """Global tag list, sort by activities in recent 14 days."""
   tags(input: RecommendInput!): TagConnection!
 
-  """Hottest tag list"""
-  hottestTags(input: RecommendInput!): TagConnection!
-
-  """Selected tag list"""
-  selectedTags(input: RecommendInput!): TagConnection!
-
   """Global user list, sort by activities in recent 6 month."""
   authors(input: RecommendInput!): UserConnection!
+}
 
-  """Global circles sort by created time."""
-  newestCircles(input: ConnectionArgs!): CircleConnection!
-
-  """Global circles sort by latest activity time."""
-  hottestCircles(input: ConnectionArgs!): CircleConnection!
+input ChannelArticlesInput {
+  channelId: ID!
+  after: String
+  first: Int
 }
 
 input RecommendInput {
@@ -2697,6 +2724,7 @@ type UserOSS {
   boost: Float!
   score: Float!
   restrictions: [UserRestriction!]!
+  featureFlags: [UserFeatureFlag!]!
 }
 
 type Appreciation {
@@ -3793,4 +3821,29 @@ type Moment implements Node {
 enum MomentState {
   active
   archived
+}
+
+type Channel {
+  id: ID!
+  name(input: TranslationArgs): String!
+  description: String
+  providerId: String!
+  enabled: Boolean!
+}
+
+input PutChannelInput {
+  id: ID
+  providerId: String!
+  name: [TranslationInput!]
+  description: String
+  enabled: Boolean
+}
+
+input SetArticleChannelsInput {
+  id: ID!
+  channels: [ID!]!
+}
+
+input ClassifyArticlesChannelsInput {
+  ids: [ID!]!
 }

src/common/enums/cache.ts

@@ -33,6 +33,7 @@ export const CACHE_PREFIX = {
   EMAIL_DOMAIL_WHITELIST: 'cache-email-domain-whitelist',
   TAG_COVERS: 'cache-tag-covers',
   SPAM_THRESHOLD: 'cache-spam-threshold',
+  ARTICLE_CHANNEL_THRESHOLD: 'cache-article-channel-threshold',
 }
 
 export const DEFAULT_IPNS_LIFETIME = '7200h' // the maximum, is 300days, almost 1 year

src/common/enums/feature.ts

@@ -8,6 +8,7 @@ export const FEATURE_NAME = {
   circle_management: 'circle_management',
   circle_interact: 'circle_interact',
   spam_detection: 'spam_detection',
+  article_channel: 'article_channel',
 } as const
 
 export const FEATURE_FLAG = {