diff --git a/.github/cherrypick/test-opensearch.yml b/.github/cherrypick/test-opensearch.yml new file mode 100644 index 0000000000..4b40506885 --- /dev/null +++ b/.github/cherrypick/test-opensearch.yml @@ -0,0 +1,23 @@ +url: 'http://cherrypick.local' + +# ローカルでテストするときにポートを被らないようにするためデフォルトのものとは変える(以下同じ) +port: 61812 + +db: + host: 127.0.0.1 + port: 54312 + db: test-cherrypick + user: postgres + pass: '' +redis: + host: 127.0.0.1 + port: 56312 +id: aidx + +opensearch: + host: 127.0.0.1 + port: 59200 + user: 'admin' + pass: 'eF53xwF4NYjrcXXwZ2CHgpwFL' + ssl: false + index: 'instancename' #なんでもいい diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 5b53bfa0bd..bd37c093ca 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,12 +10,14 @@ on: # for permissions - packages/cherrypick-js/** - .github/workflows/test-backend.yml + - .github/cherrypick/test*.yml pull_request: paths: - packages/backend/** # for permissions - packages/cherrypick-js/** - .github/workflows/test-backend.yml + - .github/cherrypick/test*.yml jobs: unit: runs-on: ubuntu-latest @@ -112,3 +114,57 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./packages/backend/coverage/coverage-final.json + + opensearch-e2e: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [20.16.0] + + services: + postgres: + image: postgres:15 + ports: + - 54312:5432 + env: + POSTGRES_DB: test-cherrypick + POSTGRES_HOST_AUTH_METHOD: trust + redis: + image: redis:7 + ports: + - 56312:6379 + + steps: + - uses: actions/checkout@v4.1.1 + with: + submodules: true + - name: Runs OpenSearch + uses: esmarkowski/opensearch-github-action@v1.0.0 + with: + version: 2.12.0 + security-disabled: true + opensearch_password: eF53xwF4NYjrcXXwZ2CHgpwFL + port: 59200 + - name: Install pnpm + uses: pnpm/action-setup@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4.0.3 + with: + node-version: ${{ matrix.node-version }} + cache: 'pnpm' + - run: corepack enable + - run: pnpm i --frozen-lockfile + - name: Check pnpm-lock.yaml + run: git diff --exit-code pnpm-lock.yaml + - name: Copy Configure + run: cp .github/cherrypick/test-opensearch.yml .config/test.yml + - name: Build + run: pnpm build + - name: Test + run: pnpm --filter backend test-and-coverage:e2e + - name: Upload to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./packages/backend/coverage/coverage-final.json diff --git a/CHANGELOG_YOJO.md b/CHANGELOG_YOJO.md index 7a2e244e0e..10e428170f 100644 --- a/CHANGELOG_YOJO.md +++ b/CHANGELOG_YOJO.md @@ -4,11 +4,24 @@ Cherrypick 4.11.1 ### Release Date ### General +インデックス構造が変わったため破棄して再インデックスが必要です。 +リアクションや投票が途中からインデックスにされるので再インデックスをおすすめします。 + - Enhance: 連合一覧のソートにリバーシのバージョンを追加 - Enhance: リモートのクリップをお気に入りに登録できるように - Enhance: リモートのPlayを遊べるように - Enhance: リモートのPlayをお気に入りに登録できるように - +- Feat: mastodonのindexableに対応 + - 検索で表示される条件を制限できるようになります + - 設定→プライバシーより設定できます + - 設定されている場合対応しているサーバーでは、以下のことをしたユーザーのみ検索できます + - リアクション + - リノート + - クリップ + - お気に入り + - 返信 + - 投票 + - コントロールパネル→その他で(クリップ、お気に入り、投票が)再インデックスできるようになりました ### Client - Fix: リアクションが閲覧できる状態でも見れない問題を修正 [#429](https://github.com/yojo-art/cherrypick/pull/429) - Enhance: チャートの連合グラフで割合を表示 diff --git a/locales/index.d.ts b/locales/index.d.ts index 8d1cc35645..51d83d3db5 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -3531,6 +3531,16 @@ export interface Locale extends ILocale { * オフにすると、「みつける」にアカウントが載らなくなります。 */ "makeExplorableDescription": string; + /** + * ノート検索の許可 + */ + "makeIndexable": string; + /** + * オフにすると、あなたのノートが検索で表示されにくくなります。 + * リノートやリアクションされているノートは表示されます。 + * リモートのサーバーが対応していない場合設定は無視されます。 + */ + "makeIndexableDescription": string; /** * タイムラインのノートを離して表示 */ @@ -11902,11 +11912,11 @@ export interface Locale extends ILocale { }; "_reIndexOpenSearch": { /** - * 全てのノートを再インデックスする + * 再インデックスする */ "title": string; /** - * 全てのノートを再インデックスしますか？ + * 再インデックス対象 */ "quesion": string; }; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 79a18a4a2e..bc80897ba4 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -877,6 +877,8 @@ thisIsExperimentalFeature: "これは実験的な機能です。仕様が変更 developer: "開発者" makeExplorable: "アカウントを見つけやすくする" makeExplorableDescription: "オフにすると、「みつける」にアカウントが載らなくなります。" +makeIndexable: "ノート検索の許可" +makeIndexableDescription: "オフにすると、あなたのノートが検索で表示されにくくなります。\nリノートやリアクションされているノートは表示されます。\nリモートのサーバーが対応していない場合設定は無視されます。" showGapBetweenNotesInTimeline: "タイムラインのノートを離して表示" duplicate: "複製" left: "左" @@ -3176,8 +3178,8 @@ _searchOrApShow: lookup: "照会" _reIndexOpenSearch: - title: "全てのノートを再インデックスする" - quesion: "全てのノートを再インデックスしますか？" + title: "再インデックスする" + quesion: "再インデックス対象" _reCreateOpenSearchIndex: title: "現在のインデックスを破棄して再インデックスする" diff --git a/packages/backend/migration/1726205819617-AddIsIndexable.js b/packages/backend/migration/1726205819617-AddIsIndexable.js new file mode 100644 index 0000000000..77065d6272 --- /dev/null +++ b/packages/backend/migration/1726205819617-AddIsIndexable.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project, yojo-art team + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddIsIndexable1726205819617 { + name = 'AddIsIndexable1726205819617' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "isIndexable" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "isIndexable"`); + } +} diff --git a/packages/backend/src/core/AdvancedSearchService.ts b/packages/backend/src/core/AdvancedSearchService.ts index 2dd5d414c1..309ec18e17 100644 --- a/packages/backend/src/core/AdvancedSearchService.ts +++ b/packages/backend/src/core/AdvancedSearchService.ts @@ -12,14 +12,13 @@ import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; import { MiUser } from '@/models/_.js'; -import type { NotesRepository, UsersRepository } from '@/models/_.js'; +import type { NotesRepository, UsersRepository, PollVotesRepository, PollsRepository, NoteReactionsRepository, ClipNotesRepository, NoteFavoritesRepository } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; -import { isReply } from '@/misc/is-reply.js'; import type Logger from '@/logger.js'; import { DriveService } from './DriveService.js'; @@ -28,11 +27,12 @@ type OpenSearchHit = { _id: string _score?: number _source:{ - id: string, + id: string userId: string visibility: string visibleUserIds?: string[] referenceUserId?: string + noteId?: string } } type K = string; @@ -79,10 +79,74 @@ function compileQuery(q: Q): string { } const retryLimit = 2; - +const noteIndexBody = { + mappings: { + properties: { + text: { + type: 'text', + analyzer: 'sudachi_analyzer', + }, + cw: { + type: 'text', + analyzer: 'sudachi_analyzer', + }, + userId: { type: 'keyword' }, + userHost: { type: 'keyword' }, + createdAt: { type: 'date' }, + tags: { type: 'keyword' }, + fileIds: { type: 'keyword' }, + visibility: { type: 'keyword' }, + visibleUserIds: { type: 'keyword' }, + replyId: { type: 'keyword' }, + renoteId: { type: 'keyword' }, + pollChoices: { + type: 'text', + analyzer: 'sudachi_analyzer', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + referenceUserId: { type: 'keyword' }, + sensitiveFileCount: { type: 'byte' }, + nonSensitiveFileCount: { type: 'byte' }, + }, + }, + settings: { + index: { + analysis: { + analyzer: { + sudachi_analyzer: { + filter: [ + 'sudachi_baseform', + 'sudachi_readingform', + 'sudachi_normalizedform', + ], + tokenizer: 'sudachi_a_tokenizer', + type: 'custom', + }, + }, + tokenizer: { + sudachi_a_tokenizer: { + type: 'sudachi_tokenizer', + additional_settings: '{"systemDict":"system_full.dic"}', + split_mode: 'A', + discard_punctuation: true, + }, + }, + }, + }, + }, +}; @Injectable() export class AdvancedSearchService { private opensearchNoteIndex: string | null = null; + private renoteIndex: string; + private reactionIndex: string; + private pollVoteIndex: string; + private favoriteIndex: string; + private logger: Logger; constructor( @@ -98,6 +162,21 @@ export class AdvancedSearchService { @Inject(DI.usersRepository) private usersRepository: UsersRepository, + @Inject(DI.pollsRepository) + private pollsRepository: PollsRepository, + + @Inject(DI.pollVotesRepository) + private pollVotesRepository: PollVotesRepository, + + @Inject(DI.noteReactionsRepository) + private noteReactionsRepository: NoteReactionsRepository, + + @Inject(DI.clipNotesRepository) + private clipNotesRepository: ClipNotesRepository, + + @Inject(DI.noteFavoritesRepository) + private noteFavoritesRepository: NoteFavoritesRepository, + @Inject(DI.redis) private redisClient: Redis.Redis, @@ -109,70 +188,109 @@ export class AdvancedSearchService { ) { this.logger = this.loggerService.getLogger('search'); if (opensearch && config.opensearch && config.opensearch.index) { - const indexname = `${config.opensearch.index}---notes`; - this.opensearchNoteIndex = indexname; + const notesIndexname = `${config.opensearch.index}---notes`; + this.renoteIndex = `${config.opensearch.index}---renotes`;//単純Renoteだけここ + this.reactionIndex = `${config.opensearch.index}---reaction`; + this.pollVoteIndex = `${config.opensearch.index}---pollvote`;// + this.favoriteIndex = `${config.opensearch.index}---favorite`;//お気に入りとclip + + this.opensearchNoteIndex = notesIndexname; + + //noteIndex + this.opensearch?.indices.exists({ + index: notesIndexname, + }).then((indexExists) => { + if (indexExists.statusCode === 404) [ + this.opensearch?.indices.create({ + index: notesIndexname, + body: noteIndexBody, + }).catch((error) => { + this.logger.error(error); + }), + ]; + }).catch((error) => { + this.logger.error(error); + }); + + //renoteIndex this.opensearch?.indices.exists({ - index: indexname, + index: this.renoteIndex, }).then((indexExists) => { - if (!indexExists) [ + if (indexExists.statusCode === 404) [ this.opensearch?.indices.create({ - index: indexname, + index: this.renoteIndex, body: { mappings: { properties: { - text: { - type: 'text', - analyzer: 'sudachi_analyzer' }, - cw: { - type: 'text', - analyzer: 'sudachi_analyzer' }, + renoteId: { type: 'keyword' }, userId: { type: 'keyword' }, - userHost: { type: 'keyword' }, createdAt: { type: 'date' }, - tags: { type: 'keyword' }, - fileIds: { type: 'keyword' }, - visibility: { type: 'keyword' }, - visibleUserIds: { type: 'keyword' }, - isReply: { type: 'boolean' }, - isQuote: { type: 'boolean' }, - referenceUserId: { type: 'keyword' }, - sensitiveFileCount: { type: 'byte' }, - nonSensitiveFileCount: { type: 'byte' }, }, }, - settings: { - index: { - analysis: { - analyzer: { - sudachi_analyzer: { - filter: [ - 'sudachi_baseform', - 'sudachi_readingform', - 'sudachi_normalizedform', - ], - tokenizer: 'sudachi_a_tokenizer', - type: 'custom', - }, - }, - tokenizer: { - sudachi_a_tokenizer: { - type: 'sudachi_tokenizer', - additional_settings: '{"systemDict":"system_full.dic"}', - split_mode: 'A', - discard_punctuation: true, - }, - }, - }, + }, + }), + ]; + }).catch((error) => this.logger.error(error)); + + //reactionIndex + this.opensearch?.indices.exists({ + index: this.reactionIndex, + }).then((indexExists) => { + if (indexExists.statusCode === 404) [ + this.opensearch?.indices.create({ + index: this.reactionIndex, + body: { + mappings: { + properties: { + noteId: { type: 'keyword' }, + userId: { type: 'keyword' }, + createdAt: { type: 'date' }, + reaction: { type: 'keyword' }, }, }, }, - }).catch((error) => { - this.logger.error(error); }), ]; - }).catch((error) => { - this.logger.error(error); - }); + }).catch((error) => this.logger.error(error)); + + //favoriteIndex + this.opensearch?.indices.exists({ + index: this.favoriteIndex, + }).then((indexExists) => { + if (indexExists.statusCode === 404) [ + this.opensearch?.indices.create({ + index: this.favoriteIndex, + body: { + mappings: { + properties: { + noteId: { type: 'keyword' }, + userId: { type: 'keyword' }, + clipId: { type: 'keyword' }, + }, + }, + }, + }), + ]; + }).catch((error) => this.logger.error(error)); + + //pollVoteIndex + this.opensearch?.indices.exists({ + index: this.pollVoteIndex, + }).then((indexExists) => { + if (indexExists.statusCode === 404) [ + this.opensearch?.indices.create({ + index: this.pollVoteIndex, + body: { + mappings: { + properties: { + noteId: { type: 'keyword' }, + userId: { type: 'keyword' }, + }, + }, + }, + }), + ]; + }).catch((error) => this.logger.error(error)); } else { this.logger.info('OpenSearch is not available'); this.opensearchNoteIndex = null; @@ -180,44 +298,97 @@ export class AdvancedSearchService { } @bindThis - public async indexNote(note: MiNote): Promise { - if (note.text == null && note.cw == null) return; - - if (this.opensearch) { - if (await this.redisClient.get('indexDeleted') !== null) { - return; - } - const IsReply = isReply(note); - const IsQuote = isRenote(note) && isQuote(note); - const sensitiveCount = await this.driveService.getSensitiveFileCount(note.fileIds); - const nonSensitiveCount = note.fileIds.length - sensitiveCount; - - const body = { - text: note.text, - cw: note.cw, + public async indexNote(note: MiNote, choices?: string[]): Promise { + if (!this.opensearch) return; + if (note.text == null && note.cw == null) { + if (note.userHost !== null) return;//リノートであり、ローカルユーザー + await this.index(this.renoteIndex, note.id, { + renoteId: note.renoteId, userId: note.userId, - userHost: note.userHost, createdAt: this.idService.parse(note.id).date.getTime(), - tags: note.tags, - fileIds: note.fileIds, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - isReply: IsReply, - isQuote: IsQuote, - referenceUserId: IsReply ? note.replyUserId : IsQuote ? note.renoteUserId : null, - sensitiveFileCount: sensitiveCount, - nonSensitiveFileCount: nonSensitiveCount, - }; - await this.opensearch.index({ - index: this.opensearchNoteIndex as string, - id: note.id, - body: body, - }).catch((error) => { - this.logger.error(error); }); + return; } + if (await this.redisClient.get('indexDeleted') !== null) { + return; + } + const IsQuote = isRenote(note) && isQuote(note); + const sensitiveCount = await this.driveService.getSensitiveFileCount(note.fileIds); + const nonSensitiveCount = note.fileIds.length - sensitiveCount; + + const body = { + text: note.text, + cw: note.cw, + userId: note.userId, + userHost: note.userHost, + createdAt: this.idService.parse(note.id).date.getTime(), + tags: note.tags, + fileIds: note.fileIds, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + replyId: note.replyId, + renoteId: note.renoteId, + pollChoices: choices, + referenceUserId: note.replyId ? note.replyUserId : IsQuote ? note.renoteUserId : null, + sensitiveFileCount: sensitiveCount, + nonSensitiveFileCount: nonSensitiveCount, + }; + this.index(this.opensearchNoteIndex as string, note.id, body); } + @bindThis + private async index(index: string, id: string, body: any ) { + if (!this.opensearch) return; + await this.opensearch.index({ + index: index, + id: id, + body: body, + }).catch((error) => { + this.logger.error(error); + }); + } + /** + * リアクション + */ + @bindThis + public async indexReaction(opts: { + id: string, + noteId: string, + userId: string, + reaction: string, + remote: boolean, + }) { + if (!opts.remote) { + await this.index(this.reactionIndex, opts.id, { + noteId: opts.noteId, + userId: opts.userId, + reaction: opts.reaction, + createdAt: this.idService.parse(opts.id).date.getTime(), + }); + } + } + + @bindThis + public async indexVote( + id: string, + opts: { + noteId: string; + userId: string; + }) { + await this.index(this.pollVoteIndex, id, { + noteId: opts.noteId, + userId: opts.userId, + }); + } + @bindThis + public async indexFavorite(id: string, + opts: { + noteId: string, + userId: string, + clipId?: string, + }) { + this.index(this.favoriteIndex, id, opts); + } @bindThis public async recreateIndex(): Promise { if (this.opensearch) { @@ -234,56 +405,9 @@ export class AdvancedSearchService { await this.opensearch.indices.create({ index: this.opensearchNoteIndex as string, - body: { - mappings: { - properties: { - text: { - type: 'text', - analyzer: 'sudachi_analyzer' }, - cw: { - type: 'text', - analyzer: 'sudachi_analyzer' }, - userId: { type: 'keyword' }, - userHost: { type: 'keyword' }, - createdAt: { type: 'date' }, - tags: { type: 'keyword' }, - fileIds: { type: 'keyword' }, - visibility: { type: 'keyword' }, - visibleUserIds: { type: 'keyword' }, - isReply: { type: 'boolean' }, - isQuote: { type: 'boolean' }, - referenceUserId: { type: 'keyword' }, - sensitiveFileCount: { type: 'byte' }, - nonSensitiveFileCount: { type: 'byte' }, - }, - }, - settings: { - index: { - analysis: { - analyzer: { - sudachi_analyzer: { - filter: [ - 'sudachi_baseform', - 'sudachi_readingform', - 'sudachi_normalizedform', - ], - tokenizer: 'sudachi_a_tokenizer', - type: 'custom', - }, - }, - tokenizer: { - sudachi_a_tokenizer: { - type: 'sudachi_tokenizer', - additional_settings: '{"systemDict":"system_full.dic"}', - split_mode: 'A', - discard_punctuation: true, - }, - }, - }, - }, - }, - }, - }).catch((error) => { + body: noteIndexBody, + }, + ).catch((error) => { this.logger.error(error); return; }); @@ -314,28 +438,260 @@ export class AdvancedSearchService { .limit(limit) .getMany(); notes.forEach(note => { - this.indexNote(note); + if (note.hasPoll) { + this.pollsRepository.findOneBy({ noteId: note.id }).then( (poll) => { + this.indexNote(note, poll ? poll.choices : undefined); + }); + } else { + this.indexNote(note, undefined); + } latestid = note.id; }); } this.logger.info('All notes has been indexed.'); } + @bindThis + public async fullIndexReaction(): Promise { + if (!this.opensearch) return; + + const reactionsCount = await this.noteReactionsRepository.createQueryBuilder('reac').getCount(); + const limit = 100; + let latestid = ''; + for (let index = 0; index < reactionsCount; index += limit) { + this.logger.info('indexing' + index + '/' + reactionsCount); + + const reactions = await this.noteReactionsRepository + .createQueryBuilder('reac') + .where('reac.id > :latestid', { latestid }) + .innerJoin('reac.user', 'user') + .select(['reac', 'user.host']) + .orderBy('reac.id', 'ASC') + .limit(limit) + .getMany(); + reactions.forEach(reac => { + this.indexReaction({ + id: reac.id, + noteId: reac.noteId, + userId: reac.userId, + reaction: reac.reaction, + remote: reac.user === null ? false : true, //user.host===nullなら userがnullになる + }); + latestid = reac.id; + }); + } + this.logger.info('All reactions has been indexed.'); + } + + @bindThis + public async fullIndexPollVote(): Promise { + const pollVotesCount = await this.pollVotesRepository.createQueryBuilder('pv').getCount(); + const limit = 100; + let latestid = ''; + for (let index = 0; index < pollVotesCount; index += limit) { + this.logger.info('indexing' + index + '/' + pollVotesCount); + + const votes = await this.pollVotesRepository + .createQueryBuilder('pv') + .where('pv.id > :latestid', { latestid }) + .innerJoin('pv.user', 'user') + .select(['pv', 'user.host']) + .andWhere('user.host IS NULL') + .orderBy('pv.id', 'ASC') + .limit(limit) + .getMany(); + if (votes.length === 0) { break; } + votes.forEach(pollVote => { + this.indexVote(pollVote.id, { + noteId: pollVote.noteId, + userId: pollVote.userId, + }); + latestid = pollVote.id; + }); + } + this.logger.info('All pollvotes has been indexed.'); + } + @bindThis + public async fullIndexClipNotes(): Promise { + const clipsCount = await this.clipNotesRepository.createQueryBuilder('clipnote').getCount(); + const limit = 100; + let latestid = ''; + for (let index = 0; index < clipsCount; index += limit) { + this.logger.info('indexing' + index + '/' + clipsCount); + + const clipNotes = await this.clipNotesRepository + .createQueryBuilder('clipnote') + .innerJoin('clipnote.clip', 'clip') + .select(['clipnote', 'clip.userId']) + .where('clipnote.id > :latestid', { latestid }) + .orderBy('clipnote.id', 'ASC') + .limit(limit) + .getMany(); + clipNotes.forEach(clipNote => { + this.indexFavorite(clipNote.id, { + noteId: clipNote.noteId, + userId: clipNote.clip?.userId as string, + clipId: clipNote.clipId, + }); + latestid = clipNote.id; + }); + } + } + public async fullIndexFavorites(): Promise { + const clipsCount = await this.noteFavoritesRepository.createQueryBuilder('fv').getCount(); + const limit = 100; + let latestid = ''; + for (let index = 0; index < clipsCount; index += limit) { + this.logger.info('indexing' + index + '/' + clipsCount); + + const favorites = await this.noteFavoritesRepository + .createQueryBuilder('fv') + .orderBy('fv.id', 'ASC') + .where('fv.id > :latestid', { latestid }) + .limit(limit) + .getMany(); + favorites.forEach(favorite => { + this.indexFavorite(favorite.id, { + noteId: favorite.noteId, + userId: favorite.userId, + }); + latestid = favorite.id; + }); + } + } + @bindThis + private async unindexById(index: string, id: string) { + if (!this.opensearch) return; + this.opensearch.delete({ + index: index, + id: id, + }).catch((error) => { this.logger.error(error);}); + } + + @bindThis + private async unindexByQuery(index: string, query: any) { + if (!this.opensearch) return; + this.opensearch.deleteByQuery({ + index: index, + body: { + query: query, + }, + }).catch((error) => { this.logger.error(error);}); + } + @bindThis public async unindexNote(note: MiNote): Promise { - if (this.opensearch) { - if (await this.redisClient.get('indexDeleted') !== null) { - return; - } - this.opensearch.delete({ - index: this.opensearchNoteIndex as string, - id: note.id, - }).catch((error) => { - console.error(error); + if (await this.redisClient.get('indexDeleted') !== null) { + return; + } + if (note.text == null && note.cw == null) { + //Renoteを消しとく + await this.unindexById(this.renoteIndex, note.id); + return; + } + this.unindexById(this.opensearchNoteIndex as string, note.id); + //Renoteの削除 + this.unindexByQuery(this.opensearchNoteIndex as string, { + term: { + renoteId: { + value: note.id, + }, + }, + }); + //クリップとお気に入りの削除 + this.unindexByQuery(this.favoriteIndex, { + term: { + noteId: { + value: note.id, + }, + }, + }); + //投票の削除 + this.unindexByQuery(this.pollVoteIndex, { + term: { + noteId: { + value: note.id, + }, + }, + }); + //リアクションの削除 + this.unindexByQuery(this.reactionIndex, { + term: { + noteId: { + value: note.id, + }, + }, + }); + } + + @bindThis + public async unindexReaction(id: string, remote: boolean): Promise { + if (!remote) this.unindexById(this.reactionIndex, id); + } + /** + * Favoriteだけどクリップもここ + */ + @bindThis + public async unindexFavorite(id?: string, noteId?: string, clipId?: string, userId?: string) { + if (clipId) { + this.unindexByQuery(this.favoriteIndex, { + bool: { + must: [ + { term: { noteId: { value: noteId } } }, + { term: { clipId: { value: clipId } } }, + { term: { userId: { value: userId } } }, + ], + }, + }); + } else { + this.unindexByQuery(this.favoriteIndex, { + bool: { + must: [ + { term: { userId: { value: userId } } }, + { term: { noteId: { value: noteId } } }, + ], + must_not: [ + { exists: { field: 'clipId' } }, + ], + }, }); } } + /** + * クリップが消されたときにクリップされたものを消す + */ + @bindThis + public async unindexUserClip(id: string) { + this.unindexByQuery(this.favoriteIndex, { + term: { + clipId: { + value: id, + }, + }, + }); + } + + /** + * user削除時に使う + * お気に入りとクリップの削除 + * ノートは個別で削除されるからそこで + */ + @bindThis + public async unindexUserFavorites (id: string) { + this.unindexByQuery(this.favoriteIndex, + { + term: { + userId: { + value: id, + }, + }, + }); + } + + /** + * エンドポイントから呼ばれるところ + */ @bindThis public async searchNote(q: string, me: MiUser | null, opts: { userId?: MiNote['userId'] | null; @@ -389,9 +745,9 @@ export class AdvancedSearchService { osFilter.bool.must.push({ exists: { field: 'userHost' } } ); } } - if (opts.excludeReply) osFilter.bool.must_not.push({ exists: { isReply: false } }); + if (opts.excludeReply) osFilter.bool.must_not.push({ exists: { field: 'replyId' } }); if (opts.excludeCW) osFilter.bool.must_not.push({ exists: { field: 'cw' } }); - if (opts.excludeQuote) osFilter.bool.must.push({ term: { isQuote: false } }); + if (opts.excludeQuote) osFilter.bool.must.push({ term: { field: 'renoteId' } }); if (opts.fileOption) { if (opts.fileOption === 'file-only') { osFilter.bool.must.push({ exists: { field: 'fileIds' } }); @@ -435,54 +791,22 @@ export class AdvancedSearchService { } } - if (me) { - /*ブロックされている or ミュートしているフィルタ*/ - const [ - userIdsWhoMeMuting, - userIdsWhoMeBlockingMe, - ] = await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]); - const Filter = Array.from(userIdsWhoMeMuting).concat(Array.from(userIdsWhoMeBlockingMe)); - const FollowingsCache = await this.cacheService.userFollowingsCache.fetch(me.id); - const Followings = Object.keys(FollowingsCache); - - let Option: any; - - if (opts.offset && 0 < opts.offset) { - Option = { - index: this.opensearchNoteIndex as string, - body: { - query: osFilter, - sort: [{ createdAt: { order: 'desc' } }], - }, - _source: ['id', 'userId', 'visibility', 'visibleUserIds', 'referenceUserId'], - size: pagination.limit, - from: opts.offset, - }; - } else { - Option = { - index: this.opensearchNoteIndex as string, - body: { - query: osFilter, - sort: [{ createdAt: { order: 'desc' } }], - }, - _source: ['id', 'userId', 'visibility', 'visibleUserIds', 'referenceUserId'], - size: pagination.limit, - }; - } + const Option = { + index: this.opensearchNoteIndex as string, + body: { + query: osFilter, + sort: [{ createdAt: { order: 'desc' } }], + }, + _source: me ? ['userId', 'visibility', 'visibleUserIds', 'referenceUserId'] : ['userId', 'visibility'], + size: pagination.limit, + } as any; - const Result = await this.search(Option, pagination.untilId ? 1 : 0, Filter, Followings, me.id); - if (Result.length === 0) { - return []; - } - const noteIds = Result.sort((a, b) => a._id > b._id ? -1 : 1).map((hit: any) => hit._id); - return (await this.notesRepository.findBy({ - id: In(noteIds), - })).sort((a, b) => a.id > b.id ? -1 : 1); - } else { - //meがないなら公開範囲が限られたものを探さない + if (opts.offset && 0 < opts.offset) { + Option.from = opts.offset; + } + + if (!me) { + //meがないなら公開範囲が限られたものを探さない osFilter.bool.must.push({ bool: { should: [ @@ -492,41 +816,18 @@ export class AdvancedSearchService { minimum_should_match: 1, }, }); + } - let Option: any; - if (opts.offset && 0 < opts.offset) { - Option = { - index: this.opensearchNoteIndex as string, - body: { - query: osFilter, - sort: [{ createdAt: { order: 'desc' } }], - }, - _source: ['id'], - size: pagination.limit, - from: opts.offset, - }; - } else { - Option = { - index: this.opensearchNoteIndex as string, - body: { - query: osFilter, - sort: [{ createdAt: { order: 'desc' } }], - }, - _source: ['id'], - size: pagination.limit, - }; - } + const Result = await this.search(Option, pagination.untilId ? 1 : 0, me ? me.id : undefined); + if (Result.length === 0) { + return []; + } - const Result = await this.opensearch.search(Option); + const noteIds = Result.sort((a, b) => a._id > b._id ? -1 : 1).map((hit: any) => hit._id); - if (Result.body.hits.hits.length === 0) { - return []; - } - const noteIds = Result.body.hits.hits.map((hit: any) => hit._id); - return (await this.notesRepository.findBy({ - id: In(noteIds), - })).sort((a, b) => a.id > b.id ? -1 : 1); - } + return (await this.notesRepository.findBy({ + id: In(noteIds), + })).sort((a, b) => a.id > b.id ? -1 : 1); } else { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); @@ -588,6 +889,7 @@ export class AdvancedSearchService { } this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateSearchableQuery(query, me); if (me) this.queryService.generateMutedUserQuery(query, me); if (me) this.queryService.generateBlockedUserQuery(query, me); @@ -596,95 +898,149 @@ export class AdvancedSearchService { } @bindThis - public async search( + private async search( OpenSearchOption: any, untilAvail: number, - Filter: string[], - followings: string[], - meUserId: string, + meUserId?: string, ): Promise { if (!this.opensearch) throw new Error(); - let res = await this.opensearch.search(OpenSearchOption); - let notes = res.body.hits.hits as OpenSearchHit[]; + /*ブロックされている or ミュートしているフィルタ*/ + const userIdsWhoMeMuting = meUserId ? await this.cacheService.userMutingsCache.fetch(meUserId) : new Set; + const userIdsWhoMeBlockingMe = meUserId ? await this.cacheService.userBlockedCache.fetch(meUserId) : new Set; + const Filter = Array.from(userIdsWhoMeMuting).concat(Array.from(userIdsWhoMeBlockingMe)); - if (!notes || notes.length === 0) return []; - const FilterdNotes = notes.filter( Note => {//ミュートしてるかブロックされてるので見れない - if (Filter.includes(Note._source.userId) ) return false; - if (Note._source.referenceUserId) { - if (Filter.includes(Note._source.referenceUserId)) return false; - } + let Followings = [] as string[]; + if (meUserId) { + const FollowingsCache = await this.cacheService.userFollowingsCache.fetch(meUserId); + Followings = Object.keys(FollowingsCache); + } + let notes = [] as OpenSearchHit[]; + const FilterdNotes = [] as OpenSearchHit[]; + while ( FilterdNotes.length < OpenSearchOption.size) { + const res = await this.opensearch.search(OpenSearchOption); + notes = res.body.hits.hits as OpenSearchHit[]; + if (notes.length === 0) break;//これ以上探してもない - if (['public', 'home'].includes(Note._source.visibility)) return true;//誰でも見れる + const resultPromises = notes.map(x => this.filter(x, Filter, Followings, meUserId)); + const Results = (await Promise.all(resultPromises)).filter( (x) => x !== null); - if (Note._source.visibility === 'followers') { //鍵だけどフォローしてるか自分 - if (Note._source.userId === meUserId || followings.includes(Note._source.userId)) return true; - } + if (Results.length > 0) { + const Filterd = Results.sort((a, b) => a._id > b._id ? -1 : 1); - if (Note._source.visibility === 'specified') { - if (Note._source.userId === meUserId) return true;//自分の投稿したダイレクトか自分が宛先に含まれている - if (Note._source.visibleUserIds) { - if (Note._source.visibleUserIds.includes(meUserId)) return true; + for (let i = 0; FilterdNotes.length < OpenSearchOption.size && i < Filterd.length; i++) { + FilterdNotes.push(Filterd[i]); } - } - return false; - }); - let retry = false; + } else break; - //フィルタされたノートが1件以上、最初のヒット件数が指定された数未満ではない - if (0 < (notes.length - FilterdNotes.length) && !(notes.length < OpenSearchOption.size)) { - retry = true; + if ( FilterdNotes.length === OpenSearchOption.size) break; + + //until指定 if (untilAvail === 1) { OpenSearchOption.body.query.bool.must[0] = { range: { createdAt: { lt: this.idService.parse(notes[notes.length - 1 ]._id).date.getTime() } } }; - } else { + } else if (untilAvail === 0) { OpenSearchOption.body.query.bool.must.push({ range: { createdAt: { lt: this.idService.parse(notes[notes.length - 1 ]._id).date.getTime() } } }); - untilAvail = 0; + } else { + OpenSearchOption.body.query.bool.must[OpenSearchOption.body.query.bool.must.length - 1 ] = { range: { createdAt: { lt: this.idService.parse(notes[notes.length - 1 ]._id).date.getTime() } } }; + } + } + return FilterdNotes; + } + + @bindThis + private async filter ( + Note: OpenSearchHit, + Filter: string[], + Followings: string[], + meUserId?: string): Promise { + if (meUserId) {//ミュートしているか、ブロックされている + if (Filter.includes(Note._source.userId) ) return null; + if (Note._source.referenceUserId) { + if (Filter.includes(Note._source.referenceUserId)) return null; + } + if (Note._source.userId === meUserId) {//自分のノート + return Note; } } - if (retry) { - for (let i = 0; i < retryLimit; i++) { - res = await this.opensearch.search(OpenSearchOption); - notes = res.body.hits.hits as OpenSearchHit[]; + const user = await this.cacheService.findUserById(Note._source.userId); + if (user.isIndexable === false) { //検索許可されていないが、 + if (!meUserId || !this.opensearch) { + return null; + } + const Option = { + index: this.reactionIndex, + body: { + query: { + bool: { + must: [ + { term: { noteId: Note._id } }, + { term: { userId: meUserId } }, + ], + }, + }, + }, + _source: ['id', 'userId'], + size: 1, + } as any; - if (!notes || notes.length === 0) break;//これ以上探してもない + //リアクションしているか、 + let res = await this.opensearch.search(Option); + if (res.body.hits.total.value > 0) { + return Note; + } - const Filterd = notes.filter( Note => {//ミュートしてるかブロックされてるので見れない - if (Filter.includes(Note._source.userId) ) return false; - if (Note._source.referenceUserId) { - if (Filter.includes(Note._source.referenceUserId)) return false; - } + //投票しているか、 + Option.index = this.pollVoteIndex; + res = await this.opensearch.search(Option); + if (res.body.hits.total.value > 0) { + return Note; + } - if (['public', 'home'].includes(Note._source.visibility)) return true;//誰でも見れる + //クリップもしくはお気に入りしてるか、 + Option.index = this.favoriteIndex; + res = await this.opensearch.search(Option); + if (res.body.hits.total.value > 0) { + return Note; + } - if (Note._source.visibility === 'followers') { //鍵だけどフォローしてるか自分 - if (Note._source.userId === meUserId || followings.includes(Note._source.userId)) return true; - } + //Renoteしている + Option.index = this.renoteIndex; + Option.body.query.bool.must = [ + { term: { renoteId: Note._id } }, + { term: { userId: meUserId } }, + ]; + res = await this.opensearch.search(Option); + if (res.body.hits.total.value > 0) { + return Note; + } + //返信している + Option.index = this.opensearchNoteIndex as string; + Option.body.query.bool.must = [ + { term: { replyId: Note._id } }, + { term: { userId: meUserId } }, + ]; - if (Note._source.visibility === 'specified') { - if (Note._source.userId === meUserId) return true;//自分の投稿したダイレクトか自分が宛先に含まれている - if (Note._source.visibleUserIds) { - if (Note._source.visibleUserIds.includes(meUserId)) return true; - } - } - return false; - }); + res = await this.opensearch.search(Option); + if (res.body.hits.total.value > 0) { + return Note; + } - for (let i = 0; i < notes.length - FilterdNotes.length && i < Filterd.length; i++) { - FilterdNotes.push(Filterd[i]); - } + return null; + } - if (OpenSearchOption.size === FilterdNotes.length) { - break; - } + if (['public', 'home'].includes(Note._source.visibility)) return Note;//誰でも見れる - //until指定 - if (untilAvail === 1) { - OpenSearchOption.body.query.bool.must[0] = { range: { createdAt: { lt: this.idService.parse(notes[notes.length - 1 ]._id).date.getTime() } } }; - } else { - OpenSearchOption.body.query.bool.must[OpenSearchOption.body.query.bool.must.length - 1 ] = { range: { createdAt: { lt: this.idService.parse(notes[notes.length - 1 ]._id).date.getTime() } } }; + if (meUserId) { + if (Note._source.visibility === 'followers') { //鍵だけどフォローしてる + if (Followings.includes(Note._source.userId)) return Note; + } + + if (Note._source.visibility === 'specified') {//自分が宛先に含まれている + if (Note._source.visibleUserIds) { + if (Note._source.visibleUserIds.includes(meUserId)) return Note; } } } - return FilterdNotes; + return null; } } diff --git a/packages/backend/src/core/ClipService.ts b/packages/backend/src/core/ClipService.ts index 5f89575a81..94512ccdfd 100644 --- a/packages/backend/src/core/ClipService.ts +++ b/packages/backend/src/core/ClipService.ts @@ -21,6 +21,7 @@ import { IdService } from '@/core/IdService.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { Packed } from '@/misc/json-schema.js'; import { emojis } from '@/misc/remote-api-utils.js'; +import { AdvancedSearchService } from './AdvancedSearchService.js'; @Injectable() export class ClipService { @@ -50,6 +51,7 @@ export class ClipService { private remoteUserResolveService: RemoteUserResolveService, private roleService: RoleService, private idService: IdService, + private advancedSearchService: AdvancedSearchService, ) { } @@ -103,6 +105,7 @@ export class ClipService { } await this.clipsRepository.delete(clip.id); + await this.advancedSearchService.unindexUserClip(clip.id); } @bindThis @@ -124,11 +127,21 @@ export class ClipService { } try { + const ID = this.idService.gen(); await this.clipNotesRepository.insert({ - id: this.idService.gen(), + id: ID, noteId: noteId, clipId: clip.id, }); + + await this.advancedSearchService.indexFavorite( + ID, + { + clipId: clip.id, + noteId: noteId, + userId: me.id, + }, + ); } catch (e: unknown) { if (e instanceof QueryFailedError) { if (isDuplicateKeyValueError(e)) { @@ -170,6 +183,7 @@ export class ClipService { clipId: clip.id, }); + await this.advancedSearchService.unindexFavorite(undefined, noteId, clip.id, me.id); this.notesRepository.decrement({ id: noteId }, 'clippedCount', 1); } @bindThis diff --git a/packages/backend/src/core/DeleteAccountService.ts b/packages/backend/src/core/DeleteAccountService.ts index 7f1b8f3efb..e1b91b24a7 100644 --- a/packages/backend/src/core/DeleteAccountService.ts +++ b/packages/backend/src/core/DeleteAccountService.ts @@ -13,7 +13,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; - +import { AdvancedSearchService } from './AdvancedSearchService.js'; @Injectable() export class DeleteAccountService { constructor( @@ -28,6 +28,7 @@ export class DeleteAccountService { private queueService: QueueService, private globalEventService: GlobalEventService, private moderationLogService: ModerationLogService, + private advancedSearchService: AdvancedSearchService, ) { } @@ -72,6 +73,7 @@ export class DeleteAccountService { this.queueService.deliver(user, content, inbox, true); } + this.advancedSearchService.unindexUserFavorites(user.id); this.queueService.createDeleteAccountJob(user, { soft: false, }); diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 64a9d20918..1c07593268 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -143,7 +143,7 @@ type Option = { reactionAcceptance?: MiNote['reactionAcceptance']; disableRightClick?: boolean | null; cw?: string | null; - visibility?: string; + visibility?: string | null; visibleUsers?: MinimumUser[] | null; channel?: MiChannel | null; apMentions?: MinimumUser[] | null; @@ -441,7 +441,6 @@ export class NoteCreateService implements OnApplicationShutdown { ? data.visibleUsers.map(u => u.id) : [] : [], - attachedFileTypes: data.files ? data.files.map(file => file.type) : [], // 以下非正規化データ @@ -753,7 +752,8 @@ export class NoteCreateService implements OnApplicationShutdown { } // Register to search database - this.index(note); + if (note.text !== null && note.cw !== null) this.searchService.indexNote(note);//MeiliSearch + this.advancedSearchService.indexNote(note, data.poll?.choices ?? undefined); //OpenSearch } @bindThis @@ -845,14 +845,6 @@ export class NoteCreateService implements OnApplicationShutdown { return this.apRendererService.addContext(content); } - @bindThis - private index(note: MiNote) { - if (note.text == null && note.cw == null) return; - - this.searchService.indexNote(note); - this.advancedSearchService.indexNote(note); - } - @bindThis private incNotesCountOfUser(user: { id: MiUser['id']; }) { this.usersRepository.createQueryBuilder().update() diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index c4feeaf971..2faee8a460 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -232,6 +232,52 @@ export class QueryService { } } + @bindThis + public generateSearchableQuery(q: SelectQueryBuilder, me?: { id: MiUser['id'] } | null): void { + //検索許可に基づいたクエリ生成 + if (me == null) { + //検索許可が必要 + q.andWhere('(SELECT "isIndexable" FROM "user" WHERE id = note."userId")'); + } else { + q.andWhere(new Brackets(qb => { + qb + //自分の投稿か、 + .where('note.userId = :meId') + //検索許可されている + .orWhere('(SELECT "isIndexable" FROM "user" WHERE id = note."userId")') + /* + * mastodonの検索の同じ動作をするように + + > 「公開」で送信された投稿がMastodonの検索結果にヒットするようになります。 + > ここのチェック状態にかかわらず、ほかのユーザーにブーストやお気に入り登録された投稿はそのユーザーから検索されることがあります。 + mastodonでの自分以外の indexable:false で検索可能な条件 + → ブースト済み,返信済み,投票した投稿,ブックマーク済み,お気に入り済み,ブックマーク済み + + お気に入り == APLike == リアクション + ブックマーク == クリップ | お気に入り + + * 検索は許可されていないが、 + */ + //リアクションしている + .orWhere('(EXISTS (SELECT 1 FROM "note_reaction" WHERE "noteId" = note.id AND "userId" = :meId))') + //投票している + .orWhere(new Brackets(qb => { + qb + .where('note."hasPoll" IS TRUE') + .andWhere('(EXISTS (SELECT 1 FROM "poll_vote" WHERE "noteId"=note.id AND "userId" = :meId))'); + })) + //お気に入りしている + .orWhere('(EXISTS (SELECT 1 FROM "note_favorite" WHERE "noteId" = note.id AND "userId" = :meId))') + //クリップしている + .orWhere('(EXISTS (SELECT 1 FROM "clip_note" WHERE ("clipId" = ANY (SELECT id FROM "clip" WHERE "userId" = :meId)) AND "noteId" = note.id))') + //リノートしている + .orWhere('(EXISTS (SELECT 1 FROM "note" AS r WHERE "renoteId" = note.id AND r."userId" = :meId))') + //返信している + .orWhere('(EXISTS (SELECT 1 FROM "note" AS r WHERE "replyId" = note.id AND r."userId" = :meId))'); + })); + } + } + @bindThis public generateMutedUserRenotesQueryForNotes(q: SelectQueryBuilder, me: { id: MiUser['id'] }): void { const mutingQuery = this.renoteMutingsRepository.createQueryBuilder('renote_muting') diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 371207c33a..89c64ba25b 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -30,6 +30,7 @@ import { RoleService } from '@/core/RoleService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { AdvancedSearchService } from './AdvancedSearchService.js'; const FALLBACK = '\u2764'; const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16; @@ -100,6 +101,7 @@ export class ReactionService { private apDeliverManagerService: ApDeliverManagerService, private notificationService: NotificationService, private perUserReactionsChart: PerUserReactionsChart, + private advancedSearchService: AdvancedSearchService, ) { } @@ -177,6 +179,13 @@ export class ReactionService { // Create reaction try { await this.noteReactionsRepository.insert(record); + await this.advancedSearchService.indexReaction({ + id: record.id, + noteId: record.noteId, + userId: record.userId, + reaction: record.reaction, + remote: user.host === null ? false : true, + }); } catch (e) { if (isDuplicateKeyValueError(e)) { const exists = await this.noteReactionsRepository.findOneByOrFail({ @@ -299,6 +308,7 @@ export class ReactionService { // Delete reaction const result = await this.noteReactionsRepository.delete(exist.id); + await this.advancedSearchService.unindexReaction(exist.id, user.host === null ? false : true); if (result.affected !== 1) { throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted'); diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 781085ce5c..2ac1421591 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -232,6 +232,7 @@ export class SearchService { } this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateSearchableQuery(query, me); if (me) this.queryService.generateMutedUserQuery(query, me); if (me) this.queryService.generateBlockedUserQuery(query, me); diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 2682d85f26..defeedd6c2 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -539,6 +539,7 @@ export class ApRendererService { tag, manuallyApprovesFollowers: user.isLocked, discoverable: user.isExplorable, + indexable: user.isIndexable, publicKey: this.renderKey(user, keypair, '#main-key'), isCat: user.isCat, attachment: attachment.length ? attachment : undefined, diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 60b6390a57..d3c14869b8 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -543,6 +543,7 @@ const extension_context_definition = { Emoji: 'toot:Emoji', featured: 'toot:featured', discoverable: 'toot:discoverable', + indexable: 'toot:indexable', // schema schema: 'http://schema.org#', PropertyValue: 'schema:PropertyValue', diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index e19bd700b7..92524cbed7 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -408,6 +408,7 @@ export class ApPersonService implements OnModuleInit { movedAt: person.movedTo ? new Date() : null, alsoKnownAs: person.alsoKnownAs, isExplorable: person.discoverable, + isIndexable: person.indexable ?? true, username: person.preferredUsername, usernameLower: person.preferredUsername?.toLowerCase(), host, @@ -651,6 +652,7 @@ export class ApPersonService implements OnModuleInit { const updates = { lastFetchedAt: new Date(), + isIndexable: person.indexable ?? true, inbox: person.inbox, sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox, outbox: typeof person.outbox === 'string' ? person.outbox : null, diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 289bfa7226..bd26c15a92 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -192,6 +192,7 @@ export interface IActor extends IObject { movedTo?: string; alsoKnownAs?: string[]; discoverable?: boolean; + indexable?: boolean; inbox: string; sharedInbox?: string; // 後方互換性のため publicKey?: { diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index d302f7bec2..3c189da2c7 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -547,7 +547,7 @@ export class UserEntityService implements OnModuleInit { name: r.name, iconUrl: r.iconUrl, displayOrder: r.displayOrder, - })) + })), ) : undefined, ...(isDetailed ? { @@ -566,6 +566,7 @@ export class UserEntityService implements OnModuleInit { isLocked: user.isLocked, isSilenced: this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), isSuspended: user.isSuspended, + isIndexable: user.isIndexable, description: profile!.description, location: profile!.location, birthday: profile!.birthday, diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 703cea4fcc..f8bee91dbb 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -194,6 +194,13 @@ export class MiUser { }) public isExplorable: boolean; + @Index() + @Column('boolean', { + default: true, + comment: '', + }) + public isIndexable: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/server/api/endpoints/admin/full-index.ts b/packages/backend/src/server/api/endpoints/admin/full-index.ts index b3e87bf4ea..314aa0951f 100644 --- a/packages/backend/src/server/api/endpoints/admin/full-index.ts +++ b/packages/backend/src/server/api/endpoints/admin/full-index.ts @@ -17,8 +17,13 @@ export const meta = { export const paramDef = { type: 'object', - properties: {}, - required: [], + properties: { + index: { + type: 'string', + enum: ['notes', 'reaction', 'pollVote', 'clipNotes', 'Favorites'], + }, + }, + required: ['index'], } as const; @Injectable() @@ -28,7 +33,24 @@ export default class extends Endpoint { private advancedSearchService: AdvancedSearchService, ) { super(meta, paramDef, async (ps, me) => { - await this.advancedSearchService.fullIndexNote(); - }); + switch (ps.index) { + case 'notes': + this.advancedSearchService.fullIndexNote(); + break; + case 'reaction': + this.advancedSearchService.fullIndexReaction(); + break; + case 'pollVote': + this.advancedSearchService.fullIndexPollVote(); + break; + case 'clipNotes': + this.advancedSearchService.fullIndexClipNotes(); + break; + case 'Favorites': + this.advancedSearchService.fullIndexFavorites(); + break; + } + }, + ); } } diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 36479ffc4c..3f659f199e 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -34,9 +34,9 @@ import type { Config } from '@/config.js'; import { safeForSql } from '@/misc/safe-for-sql.js'; import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; import { notificationRecieveConfig } from '@/models/json-schema/user.js'; +import { IdService } from '@/core/IdService.js'; import { ApiLoggerService } from '../../ApiLoggerService.js'; import { ApiError } from '../../error.js'; -import { IdService } from "@/core/IdService.js"; export const meta = { tags: ['account'], @@ -180,6 +180,7 @@ export const paramDef = { }, isLocked: { type: 'boolean' }, isExplorable: { type: 'boolean' }, + isIndexable: { type: 'boolean' }, hideOnlineStatus: { type: 'boolean' }, publicReactions: { type: 'boolean' }, carefulBot: { type: 'boolean' }, @@ -353,6 +354,7 @@ export default class extends Endpoint { // eslint- if (ps.notificationRecieveConfig !== undefined) profileUpdates.notificationRecieveConfig = ps.notificationRecieveConfig; if (typeof ps.isLocked === 'boolean') updates.isLocked = ps.isLocked; if (typeof ps.isExplorable === 'boolean') updates.isExplorable = ps.isExplorable; + if (typeof ps.isIndexable === 'boolean') updates.isIndexable = ps.isIndexable; if (typeof ps.hideOnlineStatus === 'boolean') updates.hideOnlineStatus = ps.hideOnlineStatus; if (typeof ps.publicReactions === 'boolean') profileUpdates.publicReactions = ps.publicReactions; if (typeof ps.isBot === 'boolean') updates.isBot = ps.isBot; diff --git a/packages/backend/src/server/api/endpoints/notes/favorites/create.ts b/packages/backend/src/server/api/endpoints/notes/favorites/create.ts index 804071b3d4..f159cc0a72 100644 --- a/packages/backend/src/server/api/endpoints/notes/favorites/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/favorites/create.ts @@ -11,6 +11,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { GetterService } from '@/server/api/GetterService.js'; import { DI } from '@/di-symbols.js'; import { AchievementService } from '@/core/AchievementService.js'; +import { AdvancedSearchService } from '@/core/AdvancedSearchService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -58,6 +59,7 @@ export default class extends Endpoint { // eslint- private idService: IdService, private getterService: GetterService, private achievementService: AchievementService, + private advancedSearchService: AdvancedSearchService, ) { super(meta, paramDef, async (ps, me) => { // Get favoritee @@ -79,12 +81,18 @@ export default class extends Endpoint { // eslint- } // Create favorite + const id = this.idService.gen(); await this.noteFavoritesRepository.insert({ - id: this.idService.gen(), + id: id, noteId: note.id, userId: me.id, }); + await this.advancedSearchService.indexFavorite(id, { + userId: me.id, + noteId: note.id, + }); + if (note.userHost == null && note.userId !== me.id) { this.achievementService.create(note.userId, 'myNoteFavorited1'); } diff --git a/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts b/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts index 2036facdba..3ccb243f63 100644 --- a/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/favorites/delete.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { GetterService } from '@/server/api/GetterService.js'; import { DI } from '@/di-symbols.js'; import type { NoteFavoritesRepository } from '@/models/_.js'; +import { AdvancedSearchService } from '@/core/AdvancedSearchService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -47,6 +48,7 @@ export default class extends Endpoint { // eslint- private noteFavoritesRepository: NoteFavoritesRepository, private getterService: GetterService, + private advancedSearchService: AdvancedSearchService, ) { super(meta, paramDef, async (ps, me) => { // Get favoritee @@ -67,6 +69,7 @@ export default class extends Endpoint { // eslint- // Delete favorite await this.noteFavoritesRepository.delete(exist.id); + await this.advancedSearchService.unindexFavorite(exist.id, note.id, undefined, me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index f33f49075b..8c2ab607fd 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -15,6 +15,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { AdvancedSearchService } from '@/core/AdvancedSearchService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -95,6 +96,7 @@ export default class extends Endpoint { // eslint- private apRendererService: ApRendererService, private globalEventService: GlobalEventService, private userBlockingService: UserBlockingService, + private advancedSearchService: AdvancedSearchService, ) { super(meta, paramDef, async (ps, me) => { const createdAt = new Date(); @@ -144,12 +146,17 @@ export default class extends Endpoint { // eslint- } // Create vote + const id = this.idService.gen(createdAt.getTime()); const vote = await this.pollVotesRepository.insertOne({ - id: this.idService.gen(createdAt.getTime()), + id: id, noteId: note.id, userId: me.id, choice: ps.choice, }); + this.advancedSearchService.indexVote(id, { + noteId: note.id, + userId: me.id, + }); // Increment votes count const index = ps.choice + 1; // In SQL, array index is 1 based diff --git a/packages/backend/test/e2e/search-notes.ts b/packages/backend/test/e2e/search-notes.ts new file mode 100644 index 0000000000..6ca32498dd --- /dev/null +++ b/packages/backend/test/e2e/search-notes.ts @@ -0,0 +1,632 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project, yojo-art team + * SPDX-License-Identifier: AGPL-3.0-only + */ + +process.env.NODE_ENV = 'test'; + +import * as assert from 'assert'; +import { text } from 'body-parser'; +import { api, post, signup, uploadUrl } from '../utils.js'; +import type * as misskey from 'cherrypick-js'; +import { query } from '@/misc/prelude/url.js'; + +describe('検索', () => { + let alice: misskey.entities.SignupResponse; + let bob: misskey.entities.SignupResponse; + let carol: misskey.entities.SignupResponse; + let dave: misskey.entities.SignupResponse; + let tom: misskey.entities.SignupResponse; + let root: misskey.entities.SignupResponse; + let aliceBlocking: misskey.entities.SignupResponse; + let aliceMuting: misskey.entities.SignupResponse; + let sensitiveFile0_1Note: misskey.entities.Note; + let sensitiveFile1_2Note: misskey.entities.Note; + let sensitiveFile2_2Note: misskey.entities.Note; + let sensitive1Id: string; + let sensitive2Id: string; + let file_Attached: misskey.entities.Note; + let nofile_Attached: misskey.entities.Note; + let daveNote: misskey.entities.Note; + let daveNoteDirect: misskey.entities.Note; + let tomNote: misskey.entities.Note; + let tomNoteDirect: misskey.entities.Note; + let reactedNote: misskey.entities.Note; + let votedNote: misskey.entities.Note; + let clipedNote: misskey.entities.Note; + let favoritedNote: misskey.entities.Note; + let renotedNote: misskey.entities.Note; + let replyedNote: misskey.entities.Note; + let mutingNote: misskey.entities.Note; + let blockingNote: misskey.entities.Note; + + beforeAll(async () => { + root = await signup({ username: 'root' }); + alice = await signup({ username: 'alice' }); + bob = await signup({ username: 'bob' }); + carol = await signup({ username: 'carol' }); + dave = await signup({ username: 'dave' }); + tom = await signup({ username: 'tom' }); + + aliceBlocking = await signup({ username: 'aliceBlocking' }); + aliceMuting = await signup({ username: 'aliceMuting' }); + await api('blocking/create', { userId: alice.id }, aliceBlocking); + await api('mute/create', { userId: aliceMuting.id }, alice); + blockingNote = await post(aliceBlocking, { text: 'blocking' }); + mutingNote = await post(aliceMuting, { text: 'muting' }); + const sensitive1 = await uploadUrl(bob, 'https://raw.githubusercontent.com/yojo-art/cherrypick/develop/packages/backend/test/resources/192.jpg'); + const sensitive2 = await uploadUrl(bob, 'https://raw.githubusercontent.com/yojo-art/cherrypick/develop/packages/backend/test/resources/192.png'); + const notSensitive = await uploadUrl(bob, 'https://raw.githubusercontent.com/yojo-art/cherrypick/develop/packages/backend/test/resources/rotate.jpg'); + sensitive1Id = sensitive1.id; + sensitive2Id = sensitive2.id; + + file_Attached = await post(bob, { + text: 'filetest', + fileIds: [notSensitive.id], + }); + nofile_Attached = await post(bob, { + text: 'filetest', + }); + + sensitiveFile0_1Note = await post(bob, { + text: 'test_sensitive', + fileIds: [notSensitive.id], + }); + sensitiveFile1_2Note = await post(bob, { + text: 'test_sensitive', + fileIds: [sensitive1.id], + }); + sensitiveFile2_2Note = await post(bob, { + text: 'test_sensitive', + fileIds: [sensitive1.id, sensitive2.id], + }); + + daveNote = await post(dave, { text: 'ff_test', visibility: 'followers' }); + daveNoteDirect = await post(dave, { text: 'ff_test', visibility: 'specified', visibleUserIds: [] }); + + await api('following/create', { userId: tom.id }, alice); + tomNote = await post(tom, { text: 'ff_test', visibility: 'followers' }); + tomNoteDirect = await post(tom, { text: '@alice ff_test', visibility: 'specified', visibleUserIds: [alice.id] }); + + reactedNote = await post(carol, { text: 'indexable_text' }); + votedNote = await post(carol, { + text: 'indexable_text', + poll: { + choices: ['1', '2'], + multiple: false, + }, + }); + clipedNote = await post(carol, { text: 'indexable_text' }); + favoritedNote = await post(carol, { text: 'indexable_text' }); + renotedNote = await post(carol, { text: 'indexable_text' }); + replyedNote = await post(carol, { text: 'indexable_text' }); + + await new Promise(resolve => setTimeout(resolve, 5000)); + }, 1000 * 60 * 2); + + test('権限がないのでエラー', async () => { + const res = await api('notes/advanced-search', { + query: 'filetest', + }, alice); + + assert.strictEqual(res.status, 400); + }); + test('検索ロールへのアサイン', async() => { + const roleres = await api('admin/roles/create', { + name: 'test', + description: '', + color: null, + iconUrl: null, + displayOrder: 0, + target: 'manual', + condFormula: {}, + isAdministrator: false, + isModerator: false, + isPublic: false, + isExplorable: false, + asBadge: false, + canEditMembersByModerator: false, + policies: { + canAdvancedSearchNotes: { + useDefault: false, + priority: 1, + value: true, + }, + canSearchNotes: { + useDefault: false, + priority: 1, + value: true, + }, + + }, + }, root); + + assert.strictEqual(roleres.status, 200); + + await new Promise(x => setTimeout(x, 2)); + + const assign = await api('admin/roles/assign', { + userId: alice.id, + roleId: roleres.body.id, + }, root); + assert.strictEqual(assign.status, 204); + }); + test('ファイルオプション:フィルタなし', async() => { + const res = await api('notes/advanced-search', { + query: 'filetest', + fileOption: 'combined', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 2); + }); + test('ファイルオプション:ファイル付きのみ', async() => { + const res = await api('notes/advanced-search', { + query: 'filetest', + fileOption: 'file-only', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 1); + + const noteIds = res.body.map( x => x.id); + + assert.strictEqual(noteIds.includes(file_Attached.id), true);//添付ありがある + assert.strictEqual(noteIds.includes(nofile_Attached.id), false);//添付なしがない + }); + test('ファイルオプション:ファイルなしのみ', async() => { + const res = await api('notes/advanced-search', { + query: 'filetest', + fileOption: 'no-file', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 1); + + const noteIds = res.body.map( x => x.id); + + assert.strictEqual(noteIds.includes(nofile_Attached.id), true);//添付なしがある + assert.strictEqual(noteIds.includes(file_Attached.id), false);//添付ありがない + }); + test('センシティブオプション:フラグ付与', async() => { + //ファイルへセンシティブフラグの付与 + const res1 = await api('drive/files/update', { + fileId: sensitive1Id, + isSensitive: true, + }, bob); + assert.strictEqual(res1.status, 200); + + const res2 = await api('drive/files/update', { + fileId: sensitive2Id, + isSensitive: true, + }, bob); + assert.strictEqual(res2.status, 200); + }); + test('センシティブオプション:フィルタなし', async() => { + const res = await api('notes/advanced-search', { + query: 'test_sensitive', + sensitiveFilter: 'combined', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 3); + }); + test('可視性 followers, specified', async() => { + const asres0 = await api('notes/advanced-search', { + query: 'ff_test', + }, alice); + assert.strictEqual(asres0.status, 200); + assert.strictEqual(Array.isArray(asres0.body), true); + + const ids = asres0.body.map((x) => x.id); + assert.strictEqual(ids.includes(tomNote.id), true); + assert.strictEqual(ids.includes(tomNoteDirect.id), true); + assert.strictEqual(ids.includes(daveNote.id), false); + assert.strictEqual(ids.includes(daveNoteDirect.id), false); + assert.strictEqual(asres0.body.length, 2); + }); + test('ミュートしてたら出ない', async() => { + const asres0 = await api('notes/advanced-search', { + query: 'muting', + }, alice); + assert.strictEqual(asres0.status, 200); + assert.strictEqual(Array.isArray(asres0.body), true); + assert.strictEqual(asres0.body.length, 0); + }); + test('ブロックされてたら出ない', async() => { + const asres0 = await api('notes/advanced-search', { + query: 'blocking', + }, alice); + assert.strictEqual(asres0.status, 200); + assert.strictEqual(Array.isArray(asres0.body), true); + assert.strictEqual(asres0.body.length, 0); + }); + /* + DB検索では未実装 別PRで出す + test('センシティブオプション:含む', async() => { + const res = await api('notes/advanced-search', { + query: 'test_sensitive', + sensitiveFilter: 'includeSensitive', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 2); + + const noteIds = res.body.map( x => x.id); + + assert.strictEqual(noteIds.includes(sensitiveFile0_1Note.id), false);//センシティブなファイルがないノートがない + //センシティブなファイルがあるノートがある + assert.strictEqual(noteIds.includes(sensitiveFile1_2Note.id), true); + assert.strictEqual(noteIds.includes(sensitiveFile2_2Note.id), true); + }); + test('センシティブオプション:除外', async() => { + const res = await api('notes/advanced-search', { + query: 'test_sensitive', + sensitiveFilter: 'withOutSensitive', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 1); + + const noteIds = res.body.map( x => x.id); + //センシティブなファイルがないノートがある + assert.strictEqual(noteIds.includes(sensitiveFile0_1Note.id), true); + //センシティブなファイルがあるノートがない + assert.strictEqual(noteIds.includes(sensitiveFile1_2Note.id), false); + assert.strictEqual(noteIds.includes(sensitiveFile2_2Note.id), false); + }); + test('センシティブオプション:全センシティブ', async() => { + const res = await api('notes/advanced-search', { + query: 'test_sensitive', + sensitiveFilter: 'withOutSensitive', + }, alice); + + assert.strictEqual(res.status, 200); + assert.strictEqual(Array.isArray(res.body), true); + assert.strictEqual(res.body.length, 1); + + const noteIds = res.body.map( x => x.id); + //センシティブなファイルがないノートがない + assert.strictEqual(noteIds.includes(sensitiveFile0_1Note.id), false); + //センシティブなファイルを含むノートがない + assert.strictEqual(noteIds.includes(sensitiveFile1_2Note.id), false); + //センシティブなファイルのみなノートがある + assert.strictEqual(noteIds.includes(sensitiveFile2_2Note.id), true); + }); + */ + test('indexable false ユーザーのノートは出てこない', async() => { + const ires = await api('i/update', { + isIndexable: false, + }, carol); + assert.strictEqual(ires.status, 200); + const asres0 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres0.status, 200); + assert.strictEqual(Array.isArray(asres0.body), true); + assert.strictEqual(asres0.body.length, 0); + }); + test('indexable false リアクションしたら出てくる', async() => { + const rres = await api('notes/reactions/create', { + reaction: '❤', + noteId: reactedNote.id, + }, alice); + assert.strictEqual(rres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres1 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres1.status, 200); + assert.strictEqual(Array.isArray(asres1.body), true); + assert.strictEqual(asres1.body.length, 1); + + const asnids1 = asres1.body.map( x => x.id); + assert.strictEqual(asnids1.includes(reactedNote.id), true); + }); + test('indexable false(通常検索) リアクションしたら出てくる', async() => { + const sres1 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(sres1.status, 200); + assert.strictEqual(Array.isArray(sres1.body), true); + assert.strictEqual(sres1.body.length, 1); + + const snids1 = sres1.body.map( x => x.id); + assert.strictEqual(snids1.includes(reactedNote.id), true); + }); + let rnId: string; + test('indexable false リノートしたら出てくる', async() => { + const rnres = await api('notes/create', { + renoteId: renotedNote.id, + }, alice); + await new Promise(resolve => setTimeout(resolve, 5000)); + assert.strictEqual(rnres.status, 200); + rnId = rnres.body.createdNote.id; + const asres2 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres2.status, 200); + assert.strictEqual(Array.isArray(asres2.body), true); + assert.strictEqual(asres2.body.length, 2); + + const asnids2 = asres2.body.map( x => x.id); + assert.strictEqual(asnids2.includes(renotedNote.id), true); + }); + test('indexable false(通常検索) リノートしたら出てくる', async() => { + const sres2 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(sres2.status, 200); + assert.strictEqual(Array.isArray(sres2.body), true); + assert.strictEqual(sres2.body.length, 2); + + const snids2 = sres2.body.map( x => x.id); + assert.strictEqual(snids2.includes(renotedNote.id), true); + }); + let replyId: string; + test('indexable false 返信したら出てくる', async() => { + const rpres = await api('notes/create', { + text: 'test', + replyId: replyedNote.id, + }, alice); + assert.strictEqual(rpres.status, 200); + replyId = rpres.body.createdNote.id; + await new Promise(resolve => setTimeout(resolve, 5000)); + + const asres3 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres3.status, 200); + assert.strictEqual(Array.isArray(asres3.body), true); + assert.strictEqual(asres3.body.length, 3); + + const asnids3 = asres3.body.map( x => x.id); + assert.strictEqual(asnids3.includes(replyedNote.id), true); + }); + test('indexable false(通常検索) 返信したら出てくる', async() => { + const sres3 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(sres3.status, 200); + assert.strictEqual(Array.isArray(sres3.body), true); + assert.strictEqual(sres3.body.length, 3); + + const snids3 = sres3.body.map( x => x.id); + assert.strictEqual(snids3.includes(replyedNote.id), true); + }); + test('indexable false お気に入りしたら出てくる', async() => { + const fvres = await api('notes/favorites/create', { + noteId: favoritedNote.id, + }, alice); + assert.strictEqual(fvres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + + const asres4 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres4.status, 200); + assert.strictEqual(Array.isArray(asres4.body), true); + assert.strictEqual(asres4.body.length, 4); + + const asnids4 = asres4.body.map( x => x.id); + assert.strictEqual(asnids4.includes(favoritedNote.id), true); + }); + test('indexable false(通常検索) お気に入りしたら出てくる', async() => { + const sres4 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(sres4.status, 200); + assert.strictEqual(Array.isArray(sres4.body), true); + assert.strictEqual(sres4.body.length, 4); + + const snids4 = sres4.body.map( x => x.id); + assert.strictEqual(snids4.includes(favoritedNote.id), true); + }); + let clpId: string; + test('indexable false クリップしたら出てくる', async() => { + const clpres = await api('clips/create', { + noteId: renotedNote.id, + isPublic: false, + name: 'test', + }, alice); + assert.strictEqual(clpres.status, 200); + clpId = clpres.body.id; + const clpaddres = await api('clips/add-note', { + clipId: clpres.body.id, + noteId: clipedNote.id, + }, alice); + assert.strictEqual(clpaddres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres5 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres5.status, 200); + assert.strictEqual(Array.isArray(asres5.body), true); + assert.strictEqual(asres5.body.length, 5); + + const asnids5 = asres5.body.map( x => x.id); + assert.strictEqual(asnids5.includes(clipedNote.id), true); + }); + test('indexable false(通常検索) クリップしたら出てくる', async() => { + const sres5 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(sres5.status, 200); + assert.strictEqual(Array.isArray(sres5.body), true); + assert.strictEqual(sres5.body.length, 5); + + const snids5 = sres5.body.map( x => x.id); + assert.strictEqual(snids5.includes(clipedNote.id), true); + }); + test('indexable false 投票したら出てくる', async() => { + const vres = await api('notes/polls/vote', { + noteId: votedNote.id, + choice: 0, + }, alice); + assert.strictEqual(vres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres6 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres6.status, 200); + assert.strictEqual(Array.isArray(asres6.body), true); + assert.strictEqual(asres6.body.length, 6); + + const asnids6 = asres6.body.map( x => x.id); + assert.strictEqual(asnids6.includes(votedNote.id), true); + }); + test('indexable false(通常検索) 投票したら出てくる', async() => { + const asres6 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres6.status, 200); + assert.strictEqual(Array.isArray(asres6.body), true); + assert.strictEqual(asres6.body.length, 6); + + const asnids6 = asres6.body.map( x => x.id); + assert.strictEqual(asnids6.includes(votedNote.id), true); + }); + // + test('indexable false リアクション外したらでない', async() => { + const rres = await api('notes/reactions/delete', { + noteId: reactedNote.id, + }, alice); + assert.strictEqual(rres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres1 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres1.status, 200); + assert.strictEqual(Array.isArray(asres1.body), true); + assert.strictEqual(asres1.body.length, 5); + + const asnids1 = asres1.body.map( x => x.id); + assert.strictEqual(asnids1.includes(reactedNote.id), false); + }); + test('indexable false(通常検索) リアクション外したらでない', async() => { + const asres1 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres1.status, 200); + assert.strictEqual(Array.isArray(asres1.body), true); + assert.strictEqual(asres1.body.length, 5); + + const asnids1 = asres1.body.map( x => x.id); + assert.strictEqual(asnids1.includes(reactedNote.id), false); + }); + test('indexable false リノート消したらでない', async() => { + const rnres = await api('notes/delete', { + noteId: rnId, + }, alice); + assert.strictEqual(rnres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres2 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres2.status, 200); + assert.strictEqual(Array.isArray(asres2.body), true); + assert.strictEqual(asres2.body.length, 4); + + const asnids2 = asres2.body.map( x => x.id); + assert.strictEqual(asnids2.includes(renotedNote.id), false); + }); + test('indexable false(通常検索) リノート消したらでない', async() => { + const asres2 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres2.status, 200); + assert.strictEqual(Array.isArray(asres2.body), true); + assert.strictEqual(asres2.body.length, 4); + + const asnids2 = asres2.body.map( x => x.id); + assert.strictEqual(asnids2.includes(renotedNote.id), false); + }); + test('indexable false リプライ消したらでない', async() => { + const rnres = await api('notes/delete', { + noteId: replyId, + }, alice); + assert.strictEqual(rnres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres2 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres2.status, 200); + assert.strictEqual(Array.isArray(asres2.body), true); + assert.strictEqual(asres2.body.length, 3); + + const asnids2 = asres2.body.map( x => x.id); + assert.strictEqual(asnids2.includes(renotedNote.id), false); + }); + test('indexable false(通常検索) リプライ消したらでない', async() => { + const asres2 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres2.status, 200); + assert.strictEqual(Array.isArray(asres2.body), true); + assert.strictEqual(asres2.body.length, 3); + + const asnids2 = asres2.body.map( x => x.id); + assert.strictEqual(asnids2.includes(renotedNote.id), false); + }); + test('indexable false クリップ消したらでない', async() => { + const clpaddres = await api('clips/remove-note', { + clipId: clpId, + noteId: clipedNote.id, + }, alice); + assert.strictEqual(clpaddres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + const asres5 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres5.status, 200); + assert.strictEqual(Array.isArray(asres5.body), true); + assert.strictEqual(asres5.body.length, 2); + + const asnids5 = asres5.body.map( x => x.id); + assert.strictEqual(asnids5.includes(clipedNote.id), false); + }); + test('indexable false(通常検索) クリップ消したらでない', async() => { + const asres5 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres5.status, 200); + assert.strictEqual(Array.isArray(asres5.body), true); + assert.strictEqual(asres5.body.length, 2); + + const asnids5 = asres5.body.map( x => x.id); + assert.strictEqual(asnids5.includes(clipedNote.id), false); + }); + test('indexable false お気に入り消したらでない', async() => { + const fvres = await api('notes/favorites/delete', { noteId: favoritedNote.id }, alice); + assert.strictEqual(fvres.status, 204); + await new Promise(resolve => setTimeout(resolve, 5000)); + + const asres4 = await api('notes/advanced-search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres4.status, 200); + assert.strictEqual(Array.isArray(asres4.body), true); + assert.strictEqual(asres4.body.length, 1); + + const asnids4 = asres4.body.map( x => x.id); + assert.strictEqual(asnids4.includes(favoritedNote.id), false); + }); + test('indexable false(通常検索) お気に入り消したらでない', async() => { + const asres4 = await api('notes/search', { + query: 'indexable_text', + }, alice); + assert.strictEqual(asres4.status, 200); + assert.strictEqual(Array.isArray(asres4.body), true); + assert.strictEqual(asres4.body.length, 1); + + const asnids4 = asres4.body.map( x => x.id); + assert.strictEqual(asnids4.includes(favoritedNote.id), false); + }); + //投票は消せないので対象外 +}); diff --git a/packages/backend/test/e2e/users.ts b/packages/backend/test/e2e/users.ts index 747a3314e6..18c128dbd8 100644 --- a/packages/backend/test/e2e/users.ts +++ b/packages/backend/test/e2e/users.ts @@ -7,9 +7,9 @@ process.env.NODE_ENV = 'test'; import * as assert from 'assert'; import { inspect } from 'node:util'; -import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { api, post, role, signup, successfulApiCall, uploadFile } from '../utils.js'; import type * as misskey from 'cherrypick-js'; +import { DEFAULT_POLICIES } from '@/core/RoleService.js'; describe('ユーザー', () => { // エンティティとしてのユーザーを主眼においたテストを記述する @@ -67,6 +67,7 @@ describe('ユーザー', () => { isLocked: user.isLocked, isSilenced: user.isSilenced, isSuspended: user.isSuspended, + isIndexable: user.isIndexable, description: user.description, location: user.location, birthday: user.birthday, @@ -431,6 +432,8 @@ describe('ユーザー', () => { { parameters: () => ({ isLocked: false }) }, { parameters: () => ({ isExplorable: false }) }, { parameters: () => ({ isExplorable: true }) }, + { parameters: () => ({ isIndexable: false }) }, + { parameters: () => ({ isIndexable: true }) }, { parameters: () => ({ hideOnlineStatus: true }) }, { parameters: () => ({ hideOnlineStatus: false }) }, { parameters: () => ({ publicReactions: false }) }, diff --git a/packages/cherrypick-js/etc/cherrypick-js.api.md b/packages/cherrypick-js/etc/cherrypick-js.api.md index 002dd540ce..b903e12e59 100644 --- a/packages/cherrypick-js/etc/cherrypick-js.api.md +++ b/packages/cherrypick-js/etc/cherrypick-js.api.md @@ -241,6 +241,9 @@ type AdminFederationRemoveAllFollowingRequest = operations['admin___federation__ // @public (undocumented) type AdminFederationUpdateInstanceRequest = operations['admin___federation___update-instance']['requestBody']['content']['application/json']; +// @public (undocumented) +type AdminFullIndexRequest = operations['admin___full-index']['requestBody']['content']['application/json']; + // @public (undocumented) type AdminGetIndexStatsResponse = operations['admin___get-index-stats']['responses']['200']['content']['application/json']; @@ -1291,6 +1294,7 @@ declare namespace entities { AdminResetPasswordResponse, AdminResolveAbuseUserReportRequest, AdminSendEmailRequest, + AdminFullIndexRequest, AdminServerInfoResponse, AdminShowModerationLogsRequest, AdminShowModerationLogsResponse, diff --git a/packages/cherrypick-js/src/autogen/endpoint.ts b/packages/cherrypick-js/src/autogen/endpoint.ts index 8eedc1e72e..33ce518a4c 100644 --- a/packages/cherrypick-js/src/autogen/endpoint.ts +++ b/packages/cherrypick-js/src/autogen/endpoint.ts @@ -95,6 +95,7 @@ import type { AdminResetPasswordResponse, AdminResolveAbuseUserReportRequest, AdminSendEmailRequest, + AdminFullIndexRequest, AdminServerInfoResponse, AdminShowModerationLogsRequest, AdminShowModerationLogsResponse, @@ -680,7 +681,7 @@ export type Endpoints = { 'admin/reset-password': { req: AdminResetPasswordRequest; res: AdminResetPasswordResponse }; 'admin/resolve-abuse-user-report': { req: AdminResolveAbuseUserReportRequest; res: EmptyResponse }; 'admin/send-email': { req: AdminSendEmailRequest; res: EmptyResponse }; - 'admin/full-index': { req: EmptyRequest; res: EmptyResponse }; + 'admin/full-index': { req: AdminFullIndexRequest; res: EmptyResponse }; 'admin/recreate-index': { req: EmptyRequest; res: EmptyResponse }; 'admin/server-info': { req: EmptyRequest; res: AdminServerInfoResponse }; 'admin/show-moderation-logs': { req: AdminShowModerationLogsRequest; res: AdminShowModerationLogsResponse }; diff --git a/packages/cherrypick-js/src/autogen/entities.ts b/packages/cherrypick-js/src/autogen/entities.ts index 9ca8029125..8b20f4167f 100644 --- a/packages/cherrypick-js/src/autogen/entities.ts +++ b/packages/cherrypick-js/src/autogen/entities.ts @@ -98,6 +98,7 @@ export type AdminResetPasswordRequest = operations['admin___reset-password']['re export type AdminResetPasswordResponse = operations['admin___reset-password']['responses']['200']['content']['application/json']; export type AdminResolveAbuseUserReportRequest = operations['admin___resolve-abuse-user-report']['requestBody']['content']['application/json']; export type AdminSendEmailRequest = operations['admin___send-email']['requestBody']['content']['application/json']; +export type AdminFullIndexRequest = operations['admin___full-index']['requestBody']['content']['application/json']; export type AdminServerInfoResponse = operations['admin___server-info']['responses']['200']['content']['application/json']; export type AdminShowModerationLogsRequest = operations['admin___show-moderation-logs']['requestBody']['content']['application/json']; export type AdminShowModerationLogsResponse = operations['admin___show-moderation-logs']['responses']['200']['content']['application/json']; diff --git a/packages/cherrypick-js/src/autogen/types.ts b/packages/cherrypick-js/src/autogen/types.ts index 9f6c696601..0647b4cd3c 100644 --- a/packages/cherrypick-js/src/autogen/types.ts +++ b/packages/cherrypick-js/src/autogen/types.ts @@ -9567,6 +9567,14 @@ export type operations = { * **Credential required**: *Yes* / **Permission**: *write:admin:reindex* */ 'admin___full-index': { + requestBody: { + content: { + 'application/json': { + /** @enum {string} */ + index: 'notes' | 'reaction' | 'pollVote' | 'clipNotes' | 'Favorites'; + }; + }; + }; responses: { /** @description OK (without any results) */ 204: { @@ -19931,6 +19939,7 @@ export type operations = { }[]; isLocked?: boolean; isExplorable?: boolean; + isIndexable?: boolean; hideOnlineStatus?: boolean; publicReactions?: boolean; carefulBot?: boolean; diff --git a/packages/frontend/src/components/MkUserSetupDialog.Privacy.vue b/packages/frontend/src/components/MkUserSetupDialog.Privacy.vue index 62e5d1da8a..443e39b623 100644 --- a/packages/frontend/src/components/MkUserSetupDialog.Privacy.vue +++ b/packages/frontend/src/components/MkUserSetupDialog.Privacy.vue @@ -15,6 +15,14 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.makeFollowManuallyApprove }} + + + + + + {{ i18n.ts.makeIndexable }} + + @@ -52,6 +60,7 @@ import MkFolder from '@/components/MkFolder.vue'; import { misskeyApi } from '@/scripts/misskey-api.js'; const isLocked = ref(false); +const isIndexable = ref(true); const hideOnlineStatus = ref(false); const noCrawle = ref(false); const preventAiLearning = ref(true); @@ -59,6 +68,7 @@ const preventAiLearning = ref(true); watch([isLocked, hideOnlineStatus, noCrawle, preventAiLearning], () => { misskeyApi('i/update', { isLocked: !!isLocked.value, + isIndexable: !!isIndexable.value, hideOnlineStatus: !!hideOnlineStatus.value, noCrawle: !!noCrawle.value, preventAiLearning: !!preventAiLearning.value, diff --git a/packages/frontend/src/pages/admin/other-settings.vue b/packages/frontend/src/pages/admin/other-settings.vue index eec07a602d..3336665ae9 100644 --- a/packages/frontend/src/pages/admin/other-settings.vue +++ b/packages/frontend/src/pages/admin/other-settings.vue @@ -96,15 +96,25 @@ function save() { } async function fullIndex() { - const { canceled } = await os.confirm({ - type: 'warning', - text: i18n.ts._reIndexOpenSearch.quesion, - okText: i18n.ts.yes, - cancelText: i18n.ts.no, + const { canceled, result: select } = await os.select({ + title: i18n.ts._reIndexOpenSearch.title, + items: [{ + value: 'notes', text: i18n.ts.note, + }, { + value: 'reaction', text: i18n.ts.reaction, + }, { + value: 'pollVote', text: i18n.ts.poll, + }, { + value: 'clipNotes', text: i18n.ts.clip, + }, { + value: 'Favorites', text: i18n.ts.favorite, + }], + default: 'reaction', }); - if (!canceled) { - os.apiWithDialog('admin/full-index' ); + os.apiWithDialog('admin/full-index', { + index: select, + }); } } diff --git a/packages/frontend/src/pages/settings/privacy.vue b/packages/frontend/src/pages/settings/privacy.vue index d418be624e..c6f3f6a480 100644 --- a/packages/frontend/src/pages/settings/privacy.vue +++ b/packages/frontend/src/pages/settings/privacy.vue @@ -43,6 +43,11 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.makeExplorable }} + + {{ i18n.ts.makeIndexable }} + + yojo-art +
@@ -90,6 +95,7 @@ const autoAcceptFollowed = ref($i.autoAcceptFollowed); const noCrawle = ref($i.noCrawle); const preventAiLearning = ref($i.preventAiLearning); const isExplorable = ref($i.isExplorable); +const isIndexable = ref($i.isIndexable); const hideOnlineStatus = ref($i.hideOnlineStatus); const publicReactions = ref($i.publicReactions); const followingVisibility = ref($i.followingVisibility); @@ -107,6 +113,7 @@ function save() { noCrawle: !!noCrawle.value, preventAiLearning: !!preventAiLearning.value, isExplorable: !!isExplorable.value, + isIndexable: !!isIndexable.value, hideOnlineStatus: !!hideOnlineStatus.value, publicReactions: !!publicReactions.value, followingVisibility: followingVisibility.value,