Added Vue Dashboard

webis-de · janheinrichmerker · Nov 3, 2023 · Nov 3, 2023 · Nov 5, 2023 · Nov 5, 2023
commit 6eeeae90b7bc48b56f1b080a09c9b4062ff9c1be
diff --git a/archive_query_log/dashboard/.browserslistrc b/archive_query_log/dashboard/.browserslistrc
@@ -0,0 +1,4 @@
+> 1%
+last 2 versions
+not dead
+not ie 11
diff --git a/archive_query_log/dashboard/.editorconfig b/archive_query_log/dashboard/.editorconfig
@@ -0,0 +1,5 @@
+[*.{js,jsx,ts,tsx,vue}]
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+insert_final_newline = true
diff --git a/archive_query_log/dashboard/.gitignore b/archive_query_log/dashboard/.gitignore
@@ -0,0 +1,22 @@
+.DS_Store
+node_modules
+/dist
+
+# local env files
+.env.local
+.env.*.local
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/archive_query_log/dashboard/README.md b/archive_query_log/dashboard/README.md
@@ -0,0 +1,17 @@
+# 📈 aql-monitoring
+
+Monitor and manage the crawling of the [Archive Query Log](https://github.com/webis-de/archive-query-log).
+
+## Installation
+
+```shell
+npm install
+```
+
+TODO: Add better instructions.
+
+## Usage
+
+For starting the website, navigate to the repo and do
+```npm run dev-vite```
+in your terminal
diff --git a/archive_query_log/dashboard/components.d.ts b/archive_query_log/dashboard/components.d.ts
@@ -0,0 +1,16 @@
+/* eslint-disable */
+/* prettier-ignore */
+// @ts-nocheck
+// Generated by unplugin-vue-components
+// Read more: https://github.com/vuejs/core/pull/3399
+export {}
+
+declare module 'vue' {
+  export interface GlobalComponents {
+    Footer: typeof import('./src/components/Footer.vue')['default']
+    Header: typeof import('./src/components/Header.vue')['default']
+    Home: typeof import('./src/components/Home.vue')['default']
+    ProgressTable: typeof import('./src/components/ProgressTable.vue')['default']
+    StatisticsTable: typeof import('./src/components/StatisticsTable.vue')['default']
+  }
+}
diff --git a/archive_query_log/dashboard/index.html b/archive_query_log/dashboard/index.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8" />
+    <title>Archive Query Log</title>
+</head>
+
+<body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+</body>
+
+</html>
diff --git a/archive_query_log/dashboard/package-lock.json b/archive_query_log/dashboard/package-lock.json
diff --git a/archive_query_log/dashboard/package.json b/archive_query_log/dashboard/package.json
@@ -0,0 +1,39 @@
+{
+  "name": "aql-dashboard",
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "concurrently --kill-others \"npm run dev-vite\" \"npm run dev-proxy\"",
+    "dev-vite": "vite",
+    "dev-proxy": "node proxy.js",
+    "build": "vue-tsc --noEmit && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@mdi/font": "6.2.95",
+    "axios": "^1.7.2",
+    "elastic-tiny-client": "^0.1.4",
+    "http-proxy": "^1.18.1",
+    "roboto-fontface": "*",
+    "vue": "^3.4.21",
+    "vuetify": "^3.5.8"
+  },
+  "devDependencies": {
+    "@babel/types": "^7.24.0",
+    "@types/node": "^20.11.25",
+    "@vitejs/plugin-vue": "^5.0.4",
+    "concurrently": "^8.2.2",
+    "sass": "^1.71.1",
+    "typescript": "^5.4.2",
+    "unplugin-fonts": "^1.1.1",
+    "unplugin-vue-components": "^0.26.0",
+    "vite": "^5.1.5",
+    "vite-plugin-node-polyfills": "^0.21.0",
+    "vite-plugin-vuetify": "^2.0.3",
+    "vue-tsc": "^2.0.6"
+  },
+  "description": "Monitor and manage the crawling of the [Archive Query Log](https://github.com/webis-de/archive-query-log).",
+  "main": "index.js",
+  "keywords": [],
+  "author": "",
+  "license": "ISC"
+}
diff --git a/archive_query_log/dashboard/src/App.vue b/archive_query_log/dashboard/src/App.vue
@@ -0,0 +1,81 @@
+<template>
+  <v-container class="fill-height d-flex flex-column">
+    <v-responsive class="flex-grow-1">
+      <div class="content">
+        <Header />
+        <div class="py-10"></div>
+        <StatisticsTable :statistics="statistics" />
+        <div class="py-10"></div>
+        <ProgressTable :progress="progress" />
+      </div>
+    </v-responsive>
+    <Footer />
+  </v-container>
+</template>
+
+<script>
+import Header from './components/Header.vue';
+import StatisticsTable from './components/StatisticsTable.vue';
+import ProgressTable from './components/ProgressTable.vue';
+import Footer from './components/Footer.vue';
+import { fetchData } from './client.js';
+
+export default {
+  components: {
+    Header,
+    StatisticsTable,
+    ProgressTable,
+    Footer,
+  },
+  data() {
+    return {
+      statistics: [],
+      progress: [],
+    };
+  },
+  async mounted() {
+    try {
+      this.statistics = await fetchData('/statistics');
+      this.progress = await fetchData('/progress');
+    } catch (error) {
+      console.error(error.message);
+    }
+  },
+};
+</script>
+
+<style>
+.fill-height {
+  height: 100%;
+}
+
+.d-flex {
+  display: flex;
+}
+
+.flex-column {
+  flex-direction: column;
+}
+
+.flex-grow-1 {
+  flex-grow: 1;
+}
+
+.align-center {
+  align-items: center;
+}
+
+.justify-center {
+  justify-content: center;
+}
+
+.mx-auto {
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.content {
+  padding-bottom: 80px;
+}
+</style>
+
diff --git a/archive_query_log/dashboard/src/client.js b/archive_query_log/dashboard/src/client.js
@@ -0,0 +1,13 @@
+import axios from 'axios';
+
+const BASE_URL = 'http://localhost:8000';
+
+export async function fetchData(endpoint) {
+  try {
+    const response = await axios.get(`${BASE_URL}${endpoint}`);
+    return response.data;
+  } catch (error) {
+    console.error(`Error fetching data from ${endpoint}:`, error.message);
+    throw error;
+  }
+}
diff --git a/archive_query_log/dashboard/src/components/Footer.vue b/archive_query_log/dashboard/src/components/Footer.vue
@@ -0,0 +1,18 @@
+<template>
+    <div class="footer">
+      &copy; 2024 Webis Group |
+      <a href="https://webis.de/people.html" class="white--text">Contact</a> |
+      <a href="https://webis.de/legal.html" class="white--text">Imprint</a> |
+      <a href="https://webis.de/legal.html" class="white--text">Privacy</a>
+    </div>
+  </template>
+
+  <style>
+  .footer {
+    text-align: center;
+    padding: 16px;
+    width: 100%;
+    background: #fff;
+  }
+  </style>
+
diff --git a/archive_query_log/dashboard/src/components/Header.vue b/archive_query_log/dashboard/src/components/Header.vue
@@ -0,0 +1,32 @@
+<template>
+    <div class="text-center">
+      <div class="text-body-2 font-weight mb-n1">Welcome to</div>
+      <h1 class="text-h2 font-weight-bold">Archive Query Log</h1>
+      <div class="text-body-2 font-weight-light">
+        The Archive Query Log monitoring interface.
+      </div>
+
+      <div class="py-4"></div>
+
+      <v-btn
+        text
+        href="https://github.com/webis-de/archive-query-log"
+        class="button-margin"
+        >GitHub</v-btn
+      >
+      <v-btn
+        text
+        href="https://webis.de/publications.html?q=archive#reimer_2023"
+        class="button-margin"
+        >Paper</v-btn
+      >
+      <v-btn text href="https://webis.de/">Webis</v-btn>
+    </div>
+  </template>
+
+  <style>
+  .button-margin {
+    margin-right: 100px;
+  }
+  </style>
+
diff --git a/archive_query_log/dashboard/src/components/ProgressTable.vue b/archive_query_log/dashboard/src/components/ProgressTable.vue
@@ -0,0 +1,27 @@
+<template>
+    <div>
+      <h2>Progress</h2>
+      <div class="py-4"></div>
+      <v-data-table :items="progress" item-key="input_name" items-per-page="10" class="table-shadow mx-auto">
+      </v-data-table>
+    </div>
+  </template>
+
+  <script>
+  export default {
+    props: {
+      progress: {
+        type: Array,
+        required: true,
+      },
+    },
+  };
+  </script>
+
+  <style>
+  .table-shadow {
+    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.5);
+    margin: 0 16px;
+  }
+  </style>
+
diff --git a/archive_query_log/dashboard/src/components/StatisticsTable.vue b/archive_query_log/dashboard/src/components/StatisticsTable.vue
@@ -0,0 +1,26 @@
+<template>
+    <div>
+      <h2>Statistics</h2>
+      <div class="py-4"></div>
+      <v-data-table :items="statistics" item-key="name" items-per-page="10" class="table-shadow mx-auto">
+      </v-data-table>
+    </div>
+  </template>
+
+  <script>
+  export default {
+    props: {
+      statistics: {
+        type: Array,
+        required: true,
+      },
+    },
+  };
+  </script>
+
+  <style>
+  .table-shadow {
+    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.5);
+    margin: 0 16px;
+  }
+  </style>
diff --git a/archive_query_log/dashboard/src/main.ts b/archive_query_log/dashboard/src/main.ts
@@ -0,0 +1,20 @@
+/**
+ * main.ts
+ *
+ * Bootstraps Vuetify and other plugins then mounts the App`
+ */
+
+// Plugins
+import { registerPlugins } from '@/plugins'
+
+// Components
+import App from './App.vue'
+
+// Composables
+import { createApp } from 'vue'
+
+const app = createApp(App)
+
+registerPlugins(app)
+
+app.mount('#app')
diff --git a/archive_query_log/dashboard/src/plugins/README.md b/archive_query_log/dashboard/src/plugins/README.md
@@ -0,0 +1,3 @@
+# Plugins
+
+Plugins are a way to extend the functionality of your Vue application. Use this folder for registering plugins that you want to use globally.
diff --git a/archive_query_log/dashboard/src/plugins/index.ts b/archive_query_log/dashboard/src/plugins/index.ts
@@ -0,0 +1,15 @@
+/**
+ * plugins/index.ts
+ *
+ * Automatically included in `./src/main.ts`
+ */
+
+// Plugins
+import vuetify from './vuetify'
+
+// Types
+import type { App } from 'vue'
+
+export function registerPlugins (app: App) {
+  app.use(vuetify)
+}
diff --git a/archive_query_log/dashboard/src/plugins/vuetify.ts b/archive_query_log/dashboard/src/plugins/vuetify.ts
@@ -0,0 +1,19 @@
+/**
+ * plugins/vuetify.ts
+ *
+ * Framework documentation: https://vuetifyjs.com`
+ */
+
+// Styles
+import '@mdi/font/css/materialdesignicons.css'
+import 'vuetify/styles'
+
+// Composables
+import { createVuetify } from 'vuetify'
+
+// https://vuetifyjs.com/en/introduction/why-vuetify/#feature-guides
+export default createVuetify({
+  theme: {
+    defaultTheme: 'light',
+  },
+})
diff --git a/archive_query_log/dashboard/src/vite-env.d.ts b/archive_query_log/dashboard/src/vite-env.d.ts
@@ -0,0 +1,7 @@
+/// <reference types="vite/client" />
+
+declare module '*.vue' {
+  import type { DefineComponent } from 'vue'
+  const component: DefineComponent<{}, {}, any>
+  export default component
+}
diff --git a/archive_query_log/dashboard/tsconfig.json b/archive_query_log/dashboard/tsconfig.json
@@ -0,0 +1,32 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "jsx": "preserve",
+    "lib": ["DOM", "ESNext"],
+    "baseUrl": ".",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "paths": {
+      "@/*": ["src/*"]
+    },
+    "resolveJsonModule": true,
+    "types": [
+      "vite/client",
+      "vite-plugin-vue-layouts/client",
+      "unplugin-vue-router/client"
+    ],
+    "allowJs": true,
+    "strict": true,
+    "strictNullChecks": true,
+    "noUnusedLocals": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "isolatedModules": true,
+    "skipLibCheck": true
+  },
+  "include": [
+    "./src/typed-router.d.ts"
+  ],
+  "exclude": ["dist", "node_modules", "cypress"],
+  "references": [{ "path": "./tsconfig.node.json" }],
+}
diff --git a/archive_query_log/dashboard/tsconfig.node.json b/archive_query_log/dashboard/tsconfig.node.json
@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.mts"]
+}
diff --git a/archive_query_log/dashboard/vite.config.mts b/archive_query_log/dashboard/vite.config.mts
@@ -0,0 +1,42 @@
+// Plugins
+import Components from "unplugin-vue-components/vite";
+import Vue from "@vitejs/plugin-vue";
+import Vuetify, { transformAssetUrls } from "vite-plugin-vuetify";
+import ViteFonts from "unplugin-fonts/vite";
+import { nodePolyfills } from "vite-plugin-node-polyfills";
+// Utilities
+import { defineConfig } from "vite";
+import { fileURLToPath, URL } from "node:url";
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [
+    Vue({
+      template: { transformAssetUrls },
+    }),
+    // https://github.com/vuetifyjs/vuetify-loader/tree/master/packages/vite-plugin#readme
+    Vuetify(),
+    Components(),
+    ViteFonts({
+      google: {
+        families: [
+          {
+            name: "Roboto",
+            styles: "wght@100;300;400;500;700;900",
+          },
+        ],
+      },
+    }),
+    nodePolyfills(),
+  ],
+  define: { "process.env": {} },
+  resolve: {
+    alias: {
+      "@": fileURLToPath(new URL("./src", import.meta.url)),
+    },
+    extensions: [".js", ".json", ".jsx", ".mjs", ".ts", ".tsx", ".vue"],
+  },
+  server: {
+    port: 3000,
+  },
+});
diff --git a/archive_query_log/legacy/__init__.py b/archive_query_log/legacy/__init__.py
 LOGGER: Logger = getLogger(__name__)

 if __name__ == "__main__":
    from archive_query_log.legacy.cli import main

    main()
diff --git a/archive_query_log/legacy/download/iterable.py b/archive_query_log/legacy/download/iterable.py
                record_url_header
            )
        except JSONDecodeError:
            LOGGER.warning(
                f"Could not index {record_url_header} from record "
                f"{record.rec_headers.get_header('WARC-Record-ID')}."
            )
            return None
        encoding = record.http_headers.get_header("Content-Type")
        if encoding is None:
            encoding = ""
        encoding = encoding.split(";")[-1].split("=")[-1].strip().lower()
        if encoding == "" or "/" in encoding:
            encoding = "utf8"
diff --git a/archive_query_log/legacy/model/__init__.py b/archive_query_log/legacy/model/__init__.py
        """
        MD5 hash of the original URL.
        """
        return md5(self.url.encode(), usedforsecurity=False).hexdigest()

    @cached_property
    def datetime(self) -> datetime:
diff --git a/archive_query_log/legacy/model/parse.py b/archive_query_log/legacy/model/parse.py
                ),
            )
        elif parser_type == "fragment_segment":
            from archive_query_log.legacy.queries.parse import \
                FragmentSegmentQueryParser
            return FragmentSegmentQueryParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
                ),
            )
        elif parser_type == "fragment_segment":
            from archive_query_log.legacy.queries.parse import \
                FragmentSegmentPageOffsetParser
            return FragmentSegmentPageOffsetParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
                ),
            )
        elif parser_type == "chatnoir":
            from archive_query_log.legacy.results.chatnoir import \
                ChatNoirResultsParser
            return ChatNoirResultsParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
diff --git a/archive_query_log/legacy/queries/parse.py b/archive_query_log/legacy/queries/parse.py
        if output_path.exists() and not self.overwrite:
            return
        output_path.parent.mkdir(parents=True, exist_ok=True)
        archived_urls: Iterable[ArchivedUrl] = ArchivedUrls(input_path)
        if self.verbose:
            # noinspection PyTypeChecker
            archived_urls = tqdm(
                desc="Parse SERP URLs",
                unit="URL",
            )
        archived_serp_urls_nullable = (
            self._parse_single(archived_url, focused)
            for archived_url in archived_urls
        )
            if archived_serp_url is not None
        )
        output_schema = ArchivedQueryUrl.schema()
        with (output_path.open("wb") as file,
              GzipFile(fileobj=file, mode="wb") as gzip_file,
              text_io_wrapper(gzip_file) as text_file):
            for archived_serp_url in archived_serp_urls:
            focused: bool,
    ) -> ArchivedQueryUrl | None:
        query: str | None = None
        for query_parser in self.query_parsers:
            query = query_parser.parse(archived_url)
            if query is not None:
                break

            return None

        page: int | None = None
        for page_parser in self.page_parsers:
            page = page_parser.parse(archived_url)
            if page is not None:
                break

            return None

        offset: int | None = None
        for offset_parser in self.offset_parsers:
            offset = offset_parser.parse(archived_url)
            if offset is not None:
                break

                ]

        if cdx_page is not None:
            if domain is None:
                raise RuntimeError(
                    "Domain must be specified when page is specified.")
            if len(domain_paths) < 1:
                raise RuntimeError(
                    "There must be exactly one domain path.")
            cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}.jsonl.gz"]
        else:
            domain: str | None = None,
            cdx_page: int | None = None,
    ):
        pages_list: Sequence[_CdxPage] = self._service_pages(
            data_directory=data_directory,
            focused=focused,
            service=service,
            cdx_page=cdx_page,
        )

        if len(pages_list) == 0:
            return

        pages: Iterable[_CdxPage] = pages_list
        if len(pages_list) > 1:
            # noinspection PyTypeChecker
            pages = tqdm(
                pages,
diff --git a/archive_query_log/legacy/results/parse.py b/archive_query_log/legacy/results/parse.py
        if output_path.exists() and not self.overwrite:
            return
        output_path.parent.mkdir(parents=True, exist_ok=True)
        archived_serp_contents: Iterable[ArchivedRawSerp] = (
            ArchivedRawSerps(input_path))
        if self.verbose:
            # noinspection PyTypeChecker
                desc="Parse SERP WARC records",
                unit="record",
            )
        archived_parsed_serps_nullable = (
            self.parse_single(archived_serp_content)
            for archived_serp_content in archived_serp_contents
        )
                ]

        if cdx_page is not None:
            if domain is None:
                raise RuntimeError(
                    "Domain must be specified when page is specified.")
            if len(domain_paths) < 1:
                raise RuntimeError(
                    "There must be exactly one domain path.")
            cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}"]
        else:
            domain: str | None = None,
            cdx_page: int | None = None,
    ):
        pages_list: Sequence[_CdxPage] = self._service_pages(
            data_directory=data_directory,
            focused=focused,
            service=service,
            cdx_page=cdx_page,
        )

        if len(pages_list) == 0:
            return

        pages: Iterable[_CdxPage] = pages_list
        if len(pages_list) > 1:
            # noinspection PyTypeChecker
            pages = tqdm(
                pages,
diff --git a/archive_query_log/legacy/services/__init__.py b/archive_query_log/legacy/services/__init__.py
            try:
                service = Service.schema(unknown="exclude").load(service_dict)
                services += [(service.name, service)]
            except ValidationError as e:
                if not ignore_parsing_errors:
                    raise ValueError(
                        f"Could not parse service {service_dict}") from e
        if not ignore_parsing_errors:
diff --git a/archive_query_log/legacy/urls/iterable.py b/archive_query_log/legacy/urls/iterable.py
        self._check_urls_path()

    def _check_urls_path(self):
        if not self.path.exists() or not self.path.is_file():
            raise ValueError(
                f"URLs path must be a file: {self.path}"
            )

    def __len__(self) -> int:
        with (self.path.open("rb") as file,
              GzipFile(fileobj=file, mode="rb") as gzip_file):
            return count_lines(gzip_file)

    def __iter__(self) -> Iterator[ArchivedUrl]:
        schema = ArchivedUrl.schema()
        with (self.path.open("rb") as file,
              GzipFile(fileobj=file, mode="rb") as gzip_file,
              text_io_wrapper(gzip_file) as text_file):
            for line in text_file:
                url = schema.loads(line)
                if isinstance(url, list):
                    raise ValueError(f"Expected one URL per line: {line}")
                yield url
diff --git a/archive_query_log/legacy/util/text.py b/archive_query_log/legacy/util/text.py


 def text_io_wrapper(file: IO[bytes] | IOBase) -> IO[str]:
    return TextIOWrapper(file)  # type: ignore
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Plugins

		Plugins are a way to extend the functionality of your Vue application. Use this folder for registering plugins that you want to use globally.
	LOGGER: Logger = getLogger(__name__)

	if __name__ == "__main__":
	from archive_query_log.legacy.cli import main
Check warning on line 12 in archive_query_log/legacy/__init__.py Codecov / codecov/patch archive_query_log/legacy/__init__.py#L12 `Added line #L12 was not covered by tests`

	main()
Check warning on line 14 in archive_query_log/legacy/__init__.py Codecov / codecov/patch archive_query_log/legacy/__init__.py#L14 `Added line #L14 was not covered by tests`
	record_url_header
	)
	except JSONDecodeError:
	LOGGER.warning(
Check warning on line 65 in archive_query_log/legacy/download/iterable.py Codecov / codecov/patch archive_query_log/legacy/download/iterable.py#L65 `Added line #L65 was not covered by tests`
	f"Could not index {record_url_header} from record "
	f"{record.rec_headers.get_header('WARC-Record-ID')}."
	)
	return None
	encoding = record.http_headers.get_header("Content-Type")
	if encoding is None:
	encoding = ""
Check warning on line 72 in archive_query_log/legacy/download/iterable.py Codecov / codecov/patch archive_query_log/legacy/download/iterable.py#L72 `Added line #L72 was not covered by tests`
	encoding = encoding.split(";")[-1].split("=")[-1].strip().lower()
	if encoding == "" or "/" in encoding:
	encoding = "utf8"
	"""
	MD5 hash of the original URL.
	"""
	return md5(self.url.encode(), usedforsecurity=False).hexdigest()
Check warning on line 64 in archive_query_log/legacy/model/__init__.py Codecov / codecov/patch archive_query_log/legacy/model/__init__.py#L64 `Added line #L64 was not covered by tests`

	@cached_property
	def datetime(self) -> datetime:
	),
	)
	elif parser_type == "fragment_segment":
	from archive_query_log.legacy.queries.parse import \
Check warning on line 90 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L90 `Added line #L90 was not covered by tests`
	FragmentSegmentQueryParser
	return FragmentSegmentQueryParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	),
	)
	elif parser_type == "fragment_segment":
	from archive_query_log.legacy.queries.parse import \
Check warning on line 156 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L156 `Added line #L156 was not covered by tests`
	FragmentSegmentPageOffsetParser
	return FragmentSegmentPageOffsetParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	),
	)
	elif parser_type == "chatnoir":
	from archive_query_log.legacy.results.chatnoir import \
Check warning on line 234 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L234 `Added line #L234 was not covered by tests`
	ChatNoirResultsParser
	return ChatNoirResultsParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	if output_path.exists() and not self.overwrite:
	return
	output_path.parent.mkdir(parents=True, exist_ok=True)
	archived_urls: Iterable[ArchivedUrl] = ArchivedUrls(input_path)
Check warning on line 183 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L183 `Added line #L183 was not covered by tests`
	if self.verbose:
	# noinspection PyTypeChecker
	archived_urls = tqdm(
	desc="Parse SERP URLs",
	unit="URL",
	)
	archived_serp_urls_nullable = (
Check warning on line 191 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L191 `Added line #L191 was not covered by tests`
	self._parse_single(archived_url, focused)
	for archived_url in archived_urls
	)
	if archived_serp_url is not None
	)
	output_schema = ArchivedQueryUrl.schema()
	with (output_path.open("wb") as file,
Check warning on line 201 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L201 `Added line #L201 was not covered by tests`
	GzipFile(fileobj=file, mode="wb") as gzip_file,
	text_io_wrapper(gzip_file) as text_file):
	for archived_serp_url in archived_serp_urls:
	focused: bool,
	) -> ArchivedQueryUrl \| None:
	query: str \| None = None
	for query_parser in self.query_parsers:
	query = query_parser.parse(archived_url)
Check warning on line 215 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L214-L215 `Added lines #L214 - L215 were not covered by tests`
	if query is not None:
	break

	return None

	page: int \| None = None
	for page_parser in self.page_parsers:
	page = page_parser.parse(archived_url)
Check warning on line 224 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L223-L224 `Added lines #L223 - L224 were not covered by tests`
	if page is not None:
	break

	return None

	offset: int \| None = None
	for offset_parser in self.offset_parsers:
	offset = offset_parser.parse(archived_url)
Check warning on line 233 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L232-L233 `Added lines #L232 - L233 were not covered by tests`
	if offset is not None:
	break

	]

	if cdx_page is not None:
	if domain is None:
	raise RuntimeError(
Check warning on line 284 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L283-L284 `Added lines #L283 - L284 were not covered by tests`
	"Domain must be specified when page is specified.")
	if len(domain_paths) < 1:
	raise RuntimeError(
Check warning on line 287 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L286-L287 `Added lines #L286 - L287 were not covered by tests`
	"There must be exactly one domain path.")
	cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}.jsonl.gz"]
	else:
	domain: str \| None = None,
	cdx_page: int \| None = None,
	):
	pages_list: Sequence[_CdxPage] = self._service_pages(
Check warning on line 320 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L320 `Added line #L320 was not covered by tests`
	data_directory=data_directory,
	focused=focused,
	service=service,
	cdx_page=cdx_page,
	)

	if len(pages_list) == 0:
Check warning on line 328 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L328 `Added line #L328 was not covered by tests`
	return

	pages: Iterable[_CdxPage] = pages_list
	if len(pages_list) > 1:
Check warning on line 332 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L331-L332 `Added lines #L331 - L332 were not covered by tests`
	# noinspection PyTypeChecker
	pages = tqdm(
	pages,
	try:
	service = Service.schema(unknown="exclude").load(service_dict)
	services += [(service.name, service)]
	except ValidationError as e:
	if not ignore_parsing_errors:
Check warning on line 22 in archive_query_log/legacy/services/__init__.py Codecov / codecov/patch archive_query_log/legacy/services/__init__.py#L21-L22 `Added lines #L21 - L22 were not covered by tests`
	raise ValueError(
	f"Could not parse service {service_dict}") from e
	if not ignore_parsing_errors:
	self._check_urls_path()

	def _check_urls_path(self):
	if not self.path.exists() or not self.path.is_file():
	raise ValueError(
Check warning on line 26 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L25-L26 `Added lines #L25 - L26 were not covered by tests`
	f"URLs path must be a file: {self.path}"
	)

	def __len__(self) -> int:
	with (self.path.open("rb") as file,
Check warning on line 31 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L31 `Added line #L31 was not covered by tests`
	GzipFile(fileobj=file, mode="rb") as gzip_file):
	return count_lines(gzip_file)
Check warning on line 33 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L33 `Added line #L33 was not covered by tests`

	def __iter__(self) -> Iterator[ArchivedUrl]:
	schema = ArchivedUrl.schema()
	with (self.path.open("rb") as file,
Check warning on line 37 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L36-L37 `Added lines #L36 - L37 were not covered by tests`
	GzipFile(fileobj=file, mode="rb") as gzip_file,
	text_io_wrapper(gzip_file) as text_file):
	for line in text_file:
	url = schema.loads(line)
	if isinstance(url, list):
	raise ValueError(f"Expected one URL per line: {line}")
	yield url
Check warning on line 44 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L40-L44 `Added lines #L40 - L44 were not covered by tests`


	def text_io_wrapper(file: IO[bytes] \| IOBase) -> IO[str]:
	return TextIOWrapper(file) # type: ignore
Check warning on line 19 in archive_query_log/legacy/util/text.py Codecov / codecov/patch archive_query_log/legacy/util/text.py#L19 `Added line #L19 was not covered by tests`